From 498facdf46f2f2607925653698ff423afe8005b3 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Sun, 18 Sep 2022 16:42:09 +0800 Subject: [PATCH 01/53] Influxdb protocol adapts tag schema region --- .../iotdb/influxdb/InfluxDBExample.java | 2 +- .../influxdb/integration/IoTDBInfluxDBIT.java | 7 +- .../db/protocol/influxdb/dto/IoTDBPoint.java | 7 +- .../handler/AbstractQueryHandler.java | 66 ++---- .../influxdb/handler/NewQueryHandler.java | 83 +++----- .../influxdb/handler/QueryHandler.java | 56 +----- .../influxdb/handler/QueryHandlerFactory.java | 42 ++++ .../influxdb/handler/TagQueryHandler.java | 120 +++++++++++ .../meta/AbstractInfluxDBMetaManager.java | 15 +- .../influxdb/meta/IInfluxDBMetaManager.java | 39 ++++ .../influxdb/meta/InfluxDBMetaManager.java | 50 +++++ .../meta/InfluxDBMetaManagerFactory.java | 42 ++++ .../influxdb/meta/NewInfluxDBMetaManager.java | 32 ++- .../influxdb/meta/TagInfluxDBMetaManager.java | 189 ++++++++++++++++++ .../influxdb/util/QueryResultUtils.java | 102 +++++++++- .../thrift/impl/InfluxDBServiceImpl.java | 3 +- .../thrift/impl/NewInfluxDBServiceImpl.java | 32 ++- 17 files changed, 706 insertions(+), 181 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java create mode 100644 server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java diff --git a/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java b/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java index 64a5d178226a7..cdb884dfc1b27 100644 --- a/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java +++ b/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java @@ -102,7 +102,7 @@ private static void queryData() { query = new Query( - "select count(temperature),first(temperature),last(temperature),max(temperature),mean(temperature),median(temperature),min(temperature),mode(temperature),spread(temperature),stddev(temperature),sum(temperature) from student where ((workshop=\"A1\" and production=\"B1\" and cell =\"C1\" ) or temperature< 15 )", + "select count(temperature),first(temperature),last(temperature),max(temperature),mean(temperature),median(temperature),min(temperature),mode(temperature),spread(temperature),stddev(temperature),sum(temperature) from factory where ((workshop=\"A1\" and production=\"B1\" and cell =\"C1\" ) or temperature< 15 )", database); result = influxDB.query(query); System.out.println("query2 result:" + result.getResults().get(0).getSeries().get(0).toString()); diff --git a/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java b/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java index d8c97dc2b4ab5..047cada0f145d 100644 --- a/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java +++ b/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java @@ -33,11 +33,15 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class IoTDBInfluxDBIT { @@ -164,8 +168,9 @@ public void testCommonQueryColumn() { QueryResult.Series series = result.getResults().get(0).getSeries().get(0); String[] retArray = new String[] {"time", "name", "sex", "province", "country", "score", "tel"}; + Set columnNames = new HashSet<>(Arrays.asList(retArray)); for (int i = 0; i < series.getColumns().size(); i++) { - assertEquals(retArray[i], series.getColumns().get(i)); + assertTrue(columnNames.contains(series.getColumns().get(i))); } } diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java index ac7c1ce9842e1..9f5aea7983839 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.protocol.influxdb.meta.AbstractInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.IInfluxDBMetaManager; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.utils.DataTypeUtils; import org.apache.iotdb.db.utils.ParameterUtils; @@ -59,7 +59,7 @@ public IoTDBPoint( } public IoTDBPoint( - String database, Point point, AbstractInfluxDBMetaManager metaManager, long sessionID) { + String database, Point point, IInfluxDBMetaManager influxDBMetaManager, long sessionID) { String measurement = null; Map tags = new HashMap<>(); Map fields = new HashMap<>(); @@ -105,7 +105,8 @@ public IoTDBPoint( } ParameterUtils.checkNonEmptyString(database, "database"); ParameterUtils.checkNonEmptyString(measurement, "measurement name"); - String path = metaManager.generatePath(database, measurement, tags, sessionID); + String path = + influxDBMetaManager.generatePath(database, measurement, tags, fields.keySet(), sessionID); List measurements = new ArrayList<>(); List types = new ArrayList<>(); List values = new ArrayList<>(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java index 78899f7e34bb3..10bf4dced9281 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java @@ -29,7 +29,7 @@ import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; import org.apache.iotdb.db.protocol.influxdb.function.aggregator.InfluxAggregator; import org.apache.iotdb.db.protocol.influxdb.function.selector.InfluxSelector; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxQueryOperator; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxSelectComponent; import org.apache.iotdb.db.protocol.influxdb.util.FilterUtils; @@ -40,7 +40,6 @@ import org.apache.iotdb.db.qp.logical.Operator; import org.apache.iotdb.db.qp.logical.crud.BasicFunctionOperator; import org.apache.iotdb.db.qp.logical.crud.FilterOperator; -import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxQueryResultRsp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -57,30 +56,24 @@ public abstract class AbstractQueryHandler { - abstract Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionId); - abstract InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid); + String database, String measurement, InfluxFunction function, long sessionid); abstract QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) throws AuthException; public final InfluxQueryResultRsp queryInfluxDB( - String database, - InfluxQueryOperator queryOperator, - long sessionId, - ServiceProvider serviceProvider) { + String database, InfluxQueryOperator queryOperator, long sessionId) { String measurement = queryOperator.getFromComponent().getPrefixPaths().get(0).getFullPath(); // The list of fields under the current measurement and the order of the specified rules Map fieldOrders = - getFieldOrders(database, measurement, serviceProvider, sessionId); + InfluxDBMetaManagerFactory.getInstance().getFieldOrders(database, measurement, sessionId); QueryResult queryResult; InfluxQueryResultRsp tsQueryResultRsp = new InfluxQueryResultRsp(); try { @@ -96,7 +89,6 @@ public final InfluxQueryResultRsp queryInfluxDB( : null, database, measurement, - serviceProvider, fieldOrders, sessionId); // step2 : select filter @@ -106,11 +98,7 @@ public final InfluxQueryResultRsp queryInfluxDB( else { queryResult = queryFuncWithoutFilter( - queryOperator.getSelectComponent(), - database, - measurement, - serviceProvider, - sessionId); + queryOperator.getSelectComponent(), database, measurement, sessionId); } return tsQueryResultRsp .setResultJsonString(JacksonUtils.bean2Json(queryResult)) @@ -274,18 +262,14 @@ else if (selectComponent.isHasCommonQuery()) { * @param selectComponent select data to query * @return select query result */ - public final QueryResult queryFuncWithoutFilter( - InfluxSelectComponent selectComponent, - String database, - String measurement, - ServiceProvider serviceProvider, - long sessionid) { + public QueryResult queryFuncWithoutFilter( + InfluxSelectComponent selectComponent, String database, String measurement, long sessionid) { // columns List columns = new ArrayList<>(); columns.add(InfluxSQLConstant.RESERVED_TIME); List functions = new ArrayList<>(); - String path = "root." + database + "." + measurement; + for (ResultColumn resultColumn : selectComponent.getResultColumns()) { Expression expression = resultColumn.getExpression(); if (expression instanceof FunctionExpression) { @@ -300,7 +284,7 @@ public final QueryResult queryFuncWithoutFilter( List> values = new ArrayList<>(); for (InfluxFunction function : functions) { InfluxFunctionValue functionValue = - updateByIoTDBFunc(function, serviceProvider, path, sessionid); + updateByIoTDBFunc(database, measurement, function, sessionid); // InfluxFunctionValue functionValue = function.calculateByIoTDBFunc(); if (value.size() == 0) { value.add(functionValue.getTimestamp()); @@ -330,40 +314,33 @@ public QueryResult queryExpr( FilterOperator operator, String database, String measurement, - ServiceProvider serviceProvider, Map fieldOrders, Long sessionId) throws AuthException { if (operator == null) { List expressions = new ArrayList<>(); - return queryByConditions( - expressions, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(expressions, database, measurement, fieldOrders, sessionId); } else if (operator instanceof BasicFunctionOperator) { List iExpressions = new ArrayList<>(); iExpressions.add(getIExpressionForBasicFunctionOperator((BasicFunctionOperator) operator)); - return queryByConditions( - iExpressions, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(iExpressions, database, measurement, fieldOrders, sessionId); } else { FilterOperator leftOperator = operator.getChildren().get(0); FilterOperator rightOperator = operator.getChildren().get(1); if (operator.getFilterType() == FilterConstant.FilterType.KW_OR) { return QueryResultUtils.orQueryResultProcess( - queryExpr(leftOperator, database, measurement, serviceProvider, fieldOrders, sessionId), - queryExpr( - rightOperator, database, measurement, serviceProvider, fieldOrders, sessionId)); + queryExpr(leftOperator, database, measurement, fieldOrders, sessionId), + queryExpr(rightOperator, database, measurement, fieldOrders, sessionId)); } else if (operator.getFilterType() == FilterConstant.FilterType.KW_AND) { if (canMergeOperator(leftOperator) && canMergeOperator(rightOperator)) { List iExpressions1 = getIExpressionByFilterOperatorOperator(leftOperator); List iExpressions2 = getIExpressionByFilterOperatorOperator(rightOperator); iExpressions1.addAll(iExpressions2); - return queryByConditions( - iExpressions1, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(iExpressions1, database, measurement, fieldOrders, sessionId); } else { return QueryResultUtils.andQueryResultProcess( - queryExpr( - leftOperator, database, measurement, serviceProvider, fieldOrders, sessionId), - queryExpr( - rightOperator, database, measurement, serviceProvider, fieldOrders, sessionId)); + queryExpr(leftOperator, database, measurement, fieldOrders, sessionId), + queryExpr(rightOperator, database, measurement, fieldOrders, sessionId)); } } } @@ -376,11 +353,10 @@ public QueryResult queryExpr( * @param expressions list of conditions, including tag and field condition * @return returns the results of the influxdb query */ - private QueryResult queryByConditions( + public QueryResult queryByConditions( List expressions, String database, String measurement, - ServiceProvider serviceProvider, Map fieldOrders, Long sessionId) throws AuthException { @@ -390,7 +366,8 @@ private QueryResult queryByConditions( List fieldExpressions = new ArrayList<>(); // maximum number of tags in the current query criteria int currentQueryMaxTagNum = 0; - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionId); for (IExpression expression : expressions) { SingleSeriesExpression singleSeriesExpression = ((SingleSeriesExpression) expression); // the current condition is in tag @@ -445,8 +422,7 @@ private QueryResult queryByConditions( realQuerySql += " where " + realIotDBCondition; } realQuerySql += " align by device"; - return queryByConditions( - realQuerySql, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(realQuerySql, database, measurement, null, fieldOrders, sessionId); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java index ee8d0db5c06b8..3d536df10a326 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java @@ -18,77 +18,31 @@ */ package org.apache.iotdb.db.protocol.influxdb.handler; -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.constant.InfluxSQLConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.NewInfluxDBMetaManager; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; -import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; -import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.influxdb.InfluxDBException; import org.influxdb.dto.QueryResult; -import java.util.HashMap; import java.util.List; import java.util.Map; public class NewQueryHandler extends AbstractQueryHandler { - public static TSExecuteStatementResp executeStatement(String sql, long sessionId) { - TSExecuteStatementReq tsExecuteStatementReq = new TSExecuteStatementReq(); - tsExecuteStatementReq.setStatement(sql); - tsExecuteStatementReq.setSessionId(sessionId); - tsExecuteStatementReq.setStatementId( - NewInfluxDBServiceImpl.getClientRPCService().requestStatementId(sessionId)); - tsExecuteStatementReq.setFetchSize(InfluxConstant.DEFAULT_FETCH_SIZE); - TSExecuteStatementResp executeStatementResp = - NewInfluxDBServiceImpl.getClientRPCService().executeStatement(tsExecuteStatementReq); - TSStatus tsStatus = executeStatementResp.getStatus(); - if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new InfluxDBException(tsStatus.getMessage()); - } - return executeStatementResp; - } - - @Override - public Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionID) { - Map fieldOrders = new HashMap<>(); - String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; - TSExecuteStatementResp executeStatementResp = executeStatement(showTimeseriesSql, sessionID); - List paths = QueryResultUtils.getFullPaths(executeStatementResp); - Map tagOrders = NewInfluxDBMetaManager.getTagOrders(database, measurement); - int tagOrderNums = tagOrders.size(); - int fieldNums = 0; - for (String path : paths) { - String filed = StringUtils.getFieldByPath(path); - if (!fieldOrders.containsKey(filed)) { - // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, - // and finally all fields) - fieldOrders.put(filed, tagOrderNums + fieldNums + 1); - fieldNums++; - } - } - return fieldOrders; - } - - @Override - public InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid) { + public final InfluxFunctionValue updateByIoTDBFunc( + String path, InfluxFunction function, long sessionid) { switch (function.getFunctionName()) { case InfluxSQLConstant.COUNT: { String functionSql = StringUtils.generateFunctionSql( function.getFunctionName(), function.getParmaName(), path); - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -101,7 +55,7 @@ public InfluxFunctionValue updateByIoTDBFunc( String functionSqlCount = StringUtils.generateFunctionSql("count", function.getParmaName(), path); TSExecuteStatementResp tsExecuteStatementResp = - executeStatement(functionSqlCount, sessionid); + NewInfluxDBServiceImpl.executeStatement(functionSqlCount, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -109,7 +63,8 @@ public InfluxFunctionValue updateByIoTDBFunc( } String functionSqlSum = StringUtils.generateFunctionSql("sum", function.getParmaName(), path); - tsExecuteStatementResp = executeStatement(functionSqlSum, sessionid); + tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSqlSum, sessionid); list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { function.updateValueIoTDBFunc(null, influxFunctionValue); @@ -120,7 +75,8 @@ public InfluxFunctionValue updateByIoTDBFunc( { String functionSql = StringUtils.generateFunctionSql("sum", function.getParmaName(), path); - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -142,7 +98,8 @@ public InfluxFunctionValue updateByIoTDBFunc( StringUtils.generateFunctionSql("last_value", function.getParmaName(), path); functionName = "last_value"; } - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); Map map = QueryResultUtils.getColumnNameAndValue(tsExecuteStatementResp); for (String colume : map.keySet()) { Object o = map.get(colume); @@ -152,7 +109,8 @@ public InfluxFunctionValue updateByIoTDBFunc( String.format( "select %s from %s where %s=%s", function.getParmaName(), devicePath, fullPath, o); - TSExecuteStatementResp resp = executeStatement(specificSql, sessionid); + TSExecuteStatementResp resp = + NewInfluxDBServiceImpl.executeStatement(specificSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(resp); for (InfluxFunctionValue influxFunctionValue : list) { function.updateValueIoTDBFunc(influxFunctionValue); @@ -171,7 +129,8 @@ public InfluxFunctionValue updateByIoTDBFunc( functionSql = StringUtils.generateFunctionSql("min_value", function.getParmaName(), path); } - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -185,15 +144,23 @@ public InfluxFunctionValue updateByIoTDBFunc( return function.calculateByIoTDBFunc(); } + @Override + public InfluxFunctionValue updateByIoTDBFunc( + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + "." + measurement; + return updateByIoTDBFunc(path, function, sessionid); + } + @Override public QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) { - TSExecuteStatementResp executeStatementResp = executeStatement(querySql, sessionId); + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(querySql, sessionId); return QueryResultUtils.iotdbResultConvertInfluxResult( executeStatementResp, database, measurement, fieldOrders); } diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java index b58b65f6bf5c8..3a895bcb9a2bb 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java @@ -27,14 +27,13 @@ import org.apache.iotdb.db.protocol.influxdb.constant.InfluxSQLConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; import org.apache.iotdb.db.protocol.influxdb.util.FieldUtils; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; -import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.SessionManager; +import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException; import org.apache.iotdb.tsfile.read.common.Field; @@ -49,62 +48,17 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; public class QueryHandler extends AbstractQueryHandler { - @Override - public Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionID) { - Map fieldOrders = new HashMap<>(); - long queryId = ServiceProvider.SESSION_MANAGER.requestQueryId(true); - try { - String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; - PhysicalPlan physicalPlan = - serviceProvider.getPlanner().parseSQLToPhysicalPlan(showTimeseriesSql); - QueryContext queryContext = - serviceProvider.genQueryContext( - queryId, - true, - System.currentTimeMillis(), - showTimeseriesSql, - InfluxConstant.DEFAULT_CONNECTION_TIMEOUT_MS); - QueryDataSet queryDataSet = - serviceProvider.createQueryDataSet( - queryContext, physicalPlan, InfluxConstant.DEFAULT_FETCH_SIZE); - int fieldNums = 0; - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); - int tagOrderNums = tagOrders.size(); - while (queryDataSet.hasNext()) { - List fields = queryDataSet.next().getFields(); - String filed = StringUtils.getFieldByPath(fields.get(0).getStringValue()); - if (!fieldOrders.containsKey(filed)) { - // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, - // and finally all fields) - fieldOrders.put(filed, tagOrderNums + fieldNums + 1); - fieldNums++; - } - } - } catch (QueryProcessException - | TException - | StorageEngineException - | SQLException - | IOException - | InterruptedException - | QueryFilterOptimizationException - | MetadataException e) { - throw new InfluxDBException(e.getMessage()); - } finally { - ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId); - } - return fieldOrders; - } + ServiceProvider serviceProvider = IoTDB.serviceProvider; @Override public InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid) { + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + "." + measurement; switch (function.getFunctionName()) { case InfluxSQLConstant.COUNT: { @@ -481,7 +435,7 @@ public QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) throws AuthException { diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java new file mode 100644 index 0000000000000..bfdf413a9c30b --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.handler; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; +import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; + +public class QueryHandlerFactory { + public static AbstractQueryHandler getInstance() { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getRpcImplClassName() + .equals(ClientRPCServiceImpl.class.getName())) { + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Tag: + return new TagQueryHandler(); + default: + return new NewQueryHandler(); + } + } else { + return new QueryHandler(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java new file mode 100644 index 0000000000000..0f48caa5409f3 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.handler; + +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; +import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; +import org.apache.iotdb.db.protocol.influxdb.util.FilterUtils; +import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; +import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; +import org.apache.iotdb.tsfile.read.expression.IExpression; +import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression; + +import org.influxdb.dto.QueryResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** use in tag schema region */ +public class TagQueryHandler extends NewQueryHandler { + + @Override + public InfluxFunctionValue updateByIoTDBFunc( + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + ".measurement." + measurement; + return updateByIoTDBFunc(path, function, sessionid); + } + + @Override + public QueryResult queryByConditions( + String querySql, + String database, + String measurement, + Map tagOrders, + Map fieldOrders, + long sessionId) { + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(querySql, sessionId); + return QueryResultUtils.iotdbResultConvertInfluxResult( + executeStatementResp, database, measurement, tagOrders, fieldOrders); + } + + @Override + public QueryResult queryByConditions( + List expressions, + String database, + String measurement, + Map fieldOrders, + Long sessionId) { + List fieldExpressions = new ArrayList<>(); + List tagExpressions = new ArrayList<>(); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionId); + for (IExpression expression : expressions) { + SingleSeriesExpression singleSeriesExpression = ((SingleSeriesExpression) expression); + // the current condition is in tag + if (tagOrders.containsKey(singleSeriesExpression.getSeriesPath().getFullPath())) { + tagExpressions.add(singleSeriesExpression); + } else { + fieldExpressions.add(singleSeriesExpression); + } + } + // construct the actual query path + StringBuilder curQueryPath = + new StringBuilder("root." + database + ".measurement." + measurement); + for (SingleSeriesExpression singleSeriesExpression : tagExpressions) { + String tagKey = singleSeriesExpression.getSeriesPath().getFullPath(); + String tagValue = + StringUtils.removeQuotation( + FilterUtils.getFilterStringValue(singleSeriesExpression.getFilter())); + curQueryPath.append(".").append(tagKey).append(".").append(tagValue); + } + curQueryPath.append(".**"); + + // construct actual query condition + StringBuilder realIotDBCondition = new StringBuilder(); + for (int i = 0; i < fieldExpressions.size(); i++) { + SingleSeriesExpression singleSeriesExpression = fieldExpressions.get(i); + if (i != 0) { + realIotDBCondition.append(" and "); + } + realIotDBCondition + .append(singleSeriesExpression.getSeriesPath().getFullPath()) + .append(" ") + .append((FilterUtils.getFilerSymbol(singleSeriesExpression.getFilter()))) + .append(" ") + .append(FilterUtils.getFilterStringValue(singleSeriesExpression.getFilter())); + } + // actual query SQL statement + String realQuerySql; + + realQuerySql = "select * from " + curQueryPath; + if (!(realIotDBCondition.length() == 0)) { + realQuerySql += " where " + realIotDBCondition; + } + realQuerySql += " align by device"; + return queryByConditions( + realQuerySql, database, measurement, tagOrders, fieldOrders, sessionId); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java index 513b06e59f7b6..176230f39d91d 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java @@ -22,8 +22,9 @@ import java.util.HashMap; import java.util.Map; +import java.util.Set; -public abstract class AbstractInfluxDBMetaManager { +public abstract class AbstractInfluxDBMetaManager implements IInfluxDBMetaManager { protected static final String SELECT_TAG_INFO_SQL = "select database_name,measurement_name,tag_name,tag_order from root.TAG_INFO "; @@ -32,7 +33,8 @@ public abstract class AbstractInfluxDBMetaManager { protected static Map>> database2Measurement2TagOrders = new HashMap<>(); - public static Map getTagOrders(String database, String measurement) { + @Override + public Map getTagOrders(String database, String measurement, long sessionID) { Map tagOrders = new HashMap<>(); Map> measurement2TagOrders = database2Measurement2TagOrders.get(database); @@ -45,8 +47,6 @@ public static Map getTagOrders(String database, String measurem return tagOrders; } - abstract void recover(); - abstract void setStorageGroup(String database, long sessionID); abstract void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID); @@ -69,8 +69,13 @@ public final synchronized Map getTagOrdersWithAutoCreatingSchem return createDatabase(database, sessionID).computeIfAbsent(measurement, m -> new HashMap<>()); } + @Override public final synchronized String generatePath( - String database, String measurement, Map tags, long sessionID) { + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { Map tagKeyToLayerOrders = getTagOrdersWithAutoCreatingSchema(database, measurement, sessionID); // to support rollback if fails to persisting new tag info diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java new file mode 100644 index 0000000000000..e3804ca5e76b5 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import java.util.Map; +import java.util.Set; + +/** used to manage influxdb metadata */ +public interface IInfluxDBMetaManager { + + void recover(); + + Map getFieldOrders(String database, String measurement, long sessionId); + + String generatePath( + String database, + String measurement, + Map tags, + Set fields, + long sessionID); + + Map getTagOrders(String database, String measurement, long sessionID); +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java index f2e58de977ede..e59d96dfbee3c 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java @@ -25,7 +25,10 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; import org.apache.iotdb.db.qp.Planner; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.qp.physical.sys.SetStorageGroupPlan; @@ -142,6 +145,53 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) } } + @Override + public Map getFieldOrders(String database, String measurement, long sessionID) { + Map fieldOrders = new HashMap<>(); + long queryId = ServiceProvider.SESSION_MANAGER.requestQueryId(true); + try { + String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; + PhysicalPlan physicalPlan = + serviceProvider.getPlanner().parseSQLToPhysicalPlan(showTimeseriesSql); + QueryContext queryContext = + serviceProvider.genQueryContext( + queryId, + true, + System.currentTimeMillis(), + showTimeseriesSql, + InfluxConstant.DEFAULT_CONNECTION_TIMEOUT_MS); + QueryDataSet queryDataSet = + serviceProvider.createQueryDataSet( + queryContext, physicalPlan, InfluxConstant.DEFAULT_FETCH_SIZE); + int fieldNums = 0; + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionID); + int tagOrderNums = tagOrders.size(); + while (queryDataSet.hasNext()) { + List fields = queryDataSet.next().getFields(); + String filed = StringUtils.getFieldByPath(fields.get(0).getStringValue()); + if (!fieldOrders.containsKey(filed)) { + // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, + // and finally all fields) + fieldOrders.put(filed, tagOrderNums + fieldNums + 1); + fieldNums++; + } + } + } catch (QueryProcessException + | TException + | StorageEngineException + | SQLException + | IOException + | InterruptedException + | QueryFilterOptimizationException + | MetadataException e) { + throw new InfluxDBException(e.getMessage()); + } finally { + ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId); + } + return fieldOrders; + } + private static class InfluxDBMetaManagerHolder { private static final InfluxDBMetaManager INSTANCE = new InfluxDBMetaManager(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java new file mode 100644 index 0000000000000..abad6b3e0ad08 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; +import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; + +public class InfluxDBMetaManagerFactory { + public static IInfluxDBMetaManager getInstance() { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getRpcImplClassName() + .equals(ClientRPCServiceImpl.class.getName())) { + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Tag: + return TagInfluxDBMetaManager.getInstance(); + default: + return NewInfluxDBMetaManager.getInstance(); + } + } else { + return InfluxDBMetaManager.getInstance(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java index 5269a2bf443bf..10685303c6750 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java @@ -19,8 +19,8 @@ package org.apache.iotdb.db.protocol.influxdb.meta; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.protocol.influxdb.handler.NewQueryHandler; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -57,10 +57,13 @@ public void recover() { try { TSOpenSessionResp tsOpenSessionResp = clientRPCService.openSession( - new TSOpenSessionReq().setUsername("root").setPassword("root")); + new TSOpenSessionReq() + .setUsername("root") + .setPassword("root") + .setZoneId("Asia/Shanghai")); sessionID = tsOpenSessionResp.getSessionId(); TSExecuteStatementResp resp = - NewQueryHandler.executeStatement(SELECT_TAG_INFO_SQL, sessionID); + NewInfluxDBServiceImpl.executeStatement(SELECT_TAG_INFO_SQL, sessionID); IoTDBJDBCDataSet dataSet = QueryResultUtils.creatIoTJDBCDataset(resp); try { Map> measurement2TagOrders; @@ -121,6 +124,29 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) } } + @Override + public Map getFieldOrders(String database, String measurement, long sessionID) { + Map fieldOrders = new HashMap<>(); + String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(showTimeseriesSql, sessionID); + List paths = QueryResultUtils.getFullPaths(executeStatementResp); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionID); + int tagOrderNums = tagOrders.size(); + int fieldNums = 0; + for (String path : paths) { + String filed = StringUtils.getFieldByPath(path); + if (!fieldOrders.containsKey(filed)) { + // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, + // and finally all fields) + fieldOrders.put(filed, tagOrderNums + fieldNums + 1); + fieldNums++; + } + } + return fieldOrders; + } + private static class InfluxDBMetaManagerHolder { private static final NewInfluxDBMetaManager INSTANCE = new NewInfluxDBMetaManager(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java new file mode 100644 index 0000000000000..87fd7da44e7f5 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; +import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** use in tag schema region */ +public class TagInfluxDBMetaManager implements IInfluxDBMetaManager { + private static final String STORAGE_GROUP_PATH = "root.influxdbmate"; + + private static final String TAGS_SET = "set.tags"; + + private static final String FIELDS_SET = "set.fields"; + + private TagInfluxDBMetaManager() {} + + public static TagInfluxDBMetaManager getInstance() { + return TagInfluxDBMetaManagerHolder.INSTANCE; + } + + /** use tag schema region to save state information, no need to recover here */ + @Override + public void recover() {} + + /** + * get the fields information of influxdb corresponding database and measurement through tag + * schema region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param sessionId session id + * @return field information + */ + @Override + public Map getFieldOrders(String database, String measurement, long sessionId) { + return getTimeseriesFieldOrders(database, measurement, FIELDS_SET, sessionId); + } + + /** + * convert the database,measurement,and tags of influxdb to device path of IoTDB,and save the tags + * and fields information of the database and measurement to the tag schema region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param tags influxdb tags + * @param fields influxdb fields + * @param sessionID session id + * @return device path + */ + @Override + public String generatePath( + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { + createInfluxDBMetaTimeseries(database, measurement, tags, fields, sessionID); + return generateDevicesPath(database, measurement, tags); + } + + private void createInfluxDBMetaTimeseries( + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { + List fieldsList = new ArrayList<>(tags.keySet()); + createInfluxDBMetaTimeseries(database, measurement, TAGS_SET, fieldsList, sessionID); + fieldsList.clear(); + fieldsList.addAll(fields); + createInfluxDBMetaTimeseries(database, measurement, FIELDS_SET, fieldsList, sessionID); + } + + private void createInfluxDBMetaTimeseries( + String database, String measurement, String device, List fields, long sessionID) { + String statement = generateTimeseriesStatement(database, measurement, device, fields); + NewInfluxDBServiceImpl.executeStatement(statement, sessionID); + } + + private String generateTimeseriesStatement( + String database, String measurement, String device, List fields) { + StringBuilder timeseriesStatement = + new StringBuilder( + "create aligned timeseries " + + STORAGE_GROUP_PATH + + ".database." + + database + + ".measurement." + + measurement + + "." + + device + + "("); + for (int i = 0; i < fields.size() - 1; i++) { + String field = fields.get(i); + timeseriesStatement.append(field).append(" BOOLEAN, "); + } + timeseriesStatement.append(fields.get(fields.size() - 1)).append(" BOOLEAN)"); + return timeseriesStatement.toString(); + } + + /** + * get the tags information of influxdb corresponding database and measurement through tag schema + * region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param sessionID session id + * @return tags information + */ + @Override + public Map getTagOrders(String database, String measurement, long sessionID) { + return getTimeseriesFieldOrders(database, measurement, TAGS_SET, sessionID); + } + + private Map getTimeseriesFieldOrders( + String database, String measurement, String device, long sessionID) { + TSExecuteStatementResp statementResp = + NewInfluxDBServiceImpl.executeStatement( + "show timeseries " + + STORAGE_GROUP_PATH + + ".database." + + database + + ".measurement." + + measurement + + "." + + device, + sessionID); + List timeseriesPaths = QueryResultUtils.getFullPaths(statementResp); + Map fieldOrders = new HashMap<>(); + for (String timeseriesPath : timeseriesPaths) { + String field = StringUtils.getFieldByPath(timeseriesPath); + fieldOrders.put(field, fieldOrders.size()); + } + return fieldOrders; + } + + /** + * convert the database,measurement,and tags of influxdb to device path of IoTDB,ensure that + * influxdb records with the same semantics generate the same device path, so the device path is + * generated in order after sorting the tags + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param tags influxdb tags + * @return device path + */ + private String generateDevicesPath( + String database, String measurement, Map tags) { + TreeMap tagsMap = new TreeMap<>(tags); + tagsMap.put("measurement", measurement); + StringBuilder devicePath = new StringBuilder("root." + database); + for (String tagKey : tagsMap.keySet()) { + devicePath.append(".").append(tagKey).append(".").append(tagsMap.get(tagKey)); + } + return devicePath.toString(); + } + + private static class TagInfluxDBMetaManagerHolder { + private static final TagInfluxDBMetaManager INSTANCE = new TagInfluxDBMetaManager(); + + private TagInfluxDBMetaManagerHolder() {} + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java index 325971df2598c..25199f6429f3b 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.query.dataset.AlignByDeviceDataSet; import org.apache.iotdb.rpc.IoTDBJDBCDataSet; import org.apache.iotdb.rpc.StatementExecutionException; @@ -82,7 +82,8 @@ public static QueryResult iotdbResultConvertInfluxResult( QueryResult.Series series = new QueryResult.Series(); series.setName(measurement); // gets the reverse map of the tag - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, -1); Map tagOrderReversed = tagOrders.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); @@ -294,6 +295,12 @@ public static boolean checkQueryResultNull(QueryResult queryResult) { return queryResult.getResults().get(0).getSeries() == null; } + /** + * parse time series paths from query results + * + * @param tsExecuteStatementResp query results + * @return time series paths + */ public static List getFullPaths(TSExecuteStatementResp tsExecuteStatementResp) { List res = new ArrayList<>(); IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); @@ -309,6 +316,13 @@ public static List getFullPaths(TSExecuteStatementResp tsExecuteStatemen return res; } + /** + * Convert align by device query result of NewIoTDB to the query result of influxdb,used for + * Memory and schema_file schema region + * + * @param tsExecuteStatementResp NewIoTDB execute statement resp to be converted + * @return query results in influxdb format + */ public static QueryResult iotdbResultConvertInfluxResult( TSExecuteStatementResp tsExecuteStatementResp, String database, @@ -321,7 +335,8 @@ public static QueryResult iotdbResultConvertInfluxResult( QueryResult.Series series = new QueryResult.Series(); series.setName(measurement); // gets the reverse map of the tag - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, -1); Map tagOrderReversed = tagOrders.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); @@ -381,6 +396,87 @@ public static QueryResult iotdbResultConvertInfluxResult( return queryResult; } + /** + * Convert align by device query result of NewIoTDB to the query result of influxdb,used for tag + * schema region + * + * @param tsExecuteStatementResp NewIoTDB execute statement resp to be converted + * @return query results in influxdb format + */ + public static QueryResult iotdbResultConvertInfluxResult( + TSExecuteStatementResp tsExecuteStatementResp, + String database, + String measurement, + Map tagOrders, + Map fieldOrders) { + if (tsExecuteStatementResp == null) { + return getNullQueryResult(); + } + // generate series + QueryResult.Series series = new QueryResult.Series(); + series.setName(measurement); + Map tagOrderReversed = + tagOrders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + int tagSize = tagOrderReversed.size(); + Map fieldOrdersReversed = + fieldOrders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + ArrayList tagList = new ArrayList<>(); + for (int i = 0; i < tagSize; i++) { + tagList.add(tagOrderReversed.get(i)); + } + + ArrayList fieldList = new ArrayList<>(); + for (int i = 0; i < fieldOrders.size(); i++) { + fieldList.add(fieldOrdersReversed.get(i)); + } + + ArrayList columns = new ArrayList<>(); + columns.add("time"); + columns.addAll(tagList); + columns.addAll(fieldList); + // insert columns into series + series.setColumns(columns); + List> values = new ArrayList<>(); + IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); + try { + while (ioTDBJDBCDataSet.hasCachedResults()) { + Object[] value = new Object[columns.size()]; + ioTDBJDBCDataSet.constructOneRow(); + value[0] = Long.valueOf(ioTDBJDBCDataSet.getValueByName("Time")); + String deviceName = ioTDBJDBCDataSet.getValueByName("Device"); + String[] deviceNameList = deviceName.split("\\."); + for (int i = 2; i < deviceNameList.length; i += 2) { + if (tagOrders.containsKey(deviceNameList[i])) { + int position = tagOrders.get(deviceNameList[i]) + 1; + value[position] = deviceNameList[i + 1]; + } + } + for (int i = 3; i <= ioTDBJDBCDataSet.columnNameList.size(); i++) { + Object o = ioTDBJDBCDataSet.getObject(ioTDBJDBCDataSet.findColumnNameByIndex(i)); + if (o != null) { + // insert the value of filed into it + int position = fieldOrders.get(ioTDBJDBCDataSet.findColumnNameByIndex(i)) + tagSize + 1; + value[position] = o; + } + } + values.add(Arrays.asList(value)); + } + } catch (Exception e) { + e.printStackTrace(); + } + + series.setValues(values); + + QueryResult queryResult = new QueryResult(); + QueryResult.Result result = new QueryResult.Result(); + result.setSeries(new ArrayList<>(Arrays.asList(series))); + queryResult.setResults(new ArrayList<>(Arrays.asList(result))); + + return queryResult; + } + public static List getInfluxFunctionValues( TSExecuteStatementResp tsExecuteStatementResp) { IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java index 89e5429dd19a4..8209b434876af 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java @@ -151,8 +151,7 @@ public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) { public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException { Operator operator = InfluxDBLogicalGenerator.generate(req.command); queryHandler.checkInfluxDBQueryOperator(operator); - return queryHandler.queryInfluxDB( - req.database, (InfluxQueryOperator) operator, req.sessionId, IoTDB.serviceProvider); + return queryHandler.queryInfluxDB(req.database, (InfluxQueryOperator) operator, req.sessionId); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java index 422bc27fd46af..1c40c94bc7d60 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java @@ -19,17 +19,17 @@ package org.apache.iotdb.db.service.thrift.impl; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.dto.IoTDBPoint; import org.apache.iotdb.db.protocol.influxdb.handler.AbstractQueryHandler; -import org.apache.iotdb.db.protocol.influxdb.handler.NewQueryHandler; +import org.apache.iotdb.db.protocol.influxdb.handler.QueryHandlerFactory; import org.apache.iotdb.db.protocol.influxdb.input.InfluxLineParser; -import org.apache.iotdb.db.protocol.influxdb.meta.AbstractInfluxDBMetaManager; -import org.apache.iotdb.db.protocol.influxdb.meta.NewInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.IInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxQueryOperator; import org.apache.iotdb.db.protocol.influxdb.sql.InfluxDBLogicalGenerator; import org.apache.iotdb.db.protocol.influxdb.util.InfluxReqAndRespUtils; import org.apache.iotdb.db.qp.logical.Operator; -import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.utils.DataTypeUtils; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxCloseSessionReq; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxCreateDatabaseReq; @@ -42,6 +42,8 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TSCloseSessionReq; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; @@ -57,13 +59,14 @@ public class NewInfluxDBServiceImpl implements IInfluxDBServiceWithHandler { private static final ClientRPCServiceImpl clientRPCService = new ClientRPCServiceImpl(); - private final AbstractInfluxDBMetaManager metaManager; + private final IInfluxDBMetaManager metaManager; private final AbstractQueryHandler queryHandler; public NewInfluxDBServiceImpl() { - metaManager = NewInfluxDBMetaManager.getInstance(); - queryHandler = new NewQueryHandler(); + metaManager = InfluxDBMetaManagerFactory.getInstance(); + metaManager.recover(); + queryHandler = QueryHandlerFactory.getInstance(); } public static ClientRPCServiceImpl getClientRPCService() { @@ -117,8 +120,19 @@ public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) { public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException { Operator operator = InfluxDBLogicalGenerator.generate(req.command); queryHandler.checkInfluxDBQueryOperator(operator); - return queryHandler.queryInfluxDB( - req.database, (InfluxQueryOperator) operator, req.sessionId, IoTDB.serviceProvider); + return queryHandler.queryInfluxDB(req.database, (InfluxQueryOperator) operator, req.sessionId); + } + + public static TSExecuteStatementResp executeStatement(String sql, long sessionId) { + TSExecuteStatementReq tsExecuteStatementReq = new TSExecuteStatementReq(); + tsExecuteStatementReq.setStatement(sql); + tsExecuteStatementReq.setSessionId(sessionId); + tsExecuteStatementReq.setStatementId( + NewInfluxDBServiceImpl.getClientRPCService().requestStatementId(sessionId)); + tsExecuteStatementReq.setFetchSize(InfluxConstant.DEFAULT_FETCH_SIZE); + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.getClientRPCService().executeStatement(tsExecuteStatementReq); + return executeStatementResp; } @Override From 2b28f3a96b52b92c2577e47e6c7c122e537a3586 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Sun, 18 Sep 2022 16:51:53 +0800 Subject: [PATCH 02/53] Implement mock tag schema region --- .../metadata/schemaregion/SchemaEngine.java | 6 + .../schemaregion/SchemaEngineMode.java | 3 +- .../tagschemaregion/MockTagSchemaRegion.java | 927 ++++++++++++++++++ .../common/schematree/ClusterSchemaTree.java | 66 +- 4 files changed, 993 insertions(+), 9 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java index 358d37f7b8bc2..7b3d243242dbd 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; import org.apache.iotdb.db.metadata.mtree.ConfigMTree; import org.apache.iotdb.db.metadata.rescon.SchemaResourceManager; +import org.apache.iotdb.db.metadata.schemaregion.tagschemaregion.MockTagSchemaRegion; import org.apache.iotdb.db.metadata.visitor.SchemaExecutionVisitor; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.external.api.ISeriesNumerLimiter; @@ -323,6 +324,11 @@ private ISchemaRegion createSchemaRegionWithoutExistenceCheck( new RSchemaRegionLoader() .loadRSchemaRegion(storageGroup, schemaRegionId, storageGroupMNode); break; + case Tag: + schemaRegion = + new MockTagSchemaRegion( + storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter); + break; default: throw new UnsupportedOperationException( String.format( diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java index 9147b9374c018..6b3270e87612b 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java @@ -22,5 +22,6 @@ public enum SchemaEngineMode { Memory, Schema_File, - Rocksdb_based + Rocksdb_based, + Tag } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java new file mode 100644 index 0000000000000..e4122375469c5 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java @@ -0,0 +1,927 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.schemaregion.tagschemaregion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.IDTable; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; +import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.mnode.EntityMNode; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +public class MockTagSchemaRegion implements ISchemaRegion { + + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private final String TAIL = ".**"; + private final IStorageGroupMNode storageGroupMNode; + private String storageGroupFullPath; + private SchemaRegionId schemaRegionId; + + private Map>> tagInvertedIndex; + + private List deviceIDS; + + private IDTable idTable; + + private final ISeriesNumerLimiter seriesNumerLimiter; + + public MockTagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode storageGroupMNode, + ISeriesNumerLimiter seriesNumerLimiter) + throws MetadataException { + + storageGroupFullPath = storageGroup.getFullPath(); + this.schemaRegionId = schemaRegionId; + this.storageGroupMNode = storageGroupMNode; + this.deviceIDS = new ArrayList<>(); + this.seriesNumerLimiter = seriesNumerLimiter; + tagInvertedIndex = new ConcurrentHashMap<>(); + idTable = IDTableManager.getInstance().getIDTable(storageGroup); + init(); + } + + @NotNull + private Map pathToTags(String path) { + if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); + String devicePath = path.substring(storageGroupFullPath.length() + 1); + String[] tags = devicePath.split("\\."); + Map tagsMap = new TreeMap<>(); + for (int i = 0; i < tags.length; i += 2) { + tagsMap.put(tags[i], tags[i + 1]); + } + return tagsMap; + } + + public String tagsToPath(Map tags) { + StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); + for (String tagKey : tags.keySet()) { + stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); + } + return stringBuilder.toString(); + } + + @Override + public void init() throws MetadataException { + if (!config.isEnableIDTableLogFile() + && config.getDeviceIDTransformationMethod().equals("SHA256")) { + throw new MetadataException( + "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); + } + } + + @Override + public void clear() { + return; + } + + @Override + public void forceMlog() { + return; + } + + @Override + public SchemaRegionId getSchemaRegionId() { + return schemaRegionId; + } + + @Override + public String getStorageGroupFullPath() { + return storageGroupFullPath; + } + + @Override + public void deleteSchemaRegion() throws MetadataException { + return; + } + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) { + return; + } + + private void createTagInvertedIndex(PartialPath devicePath) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + Map tagsMap = pathToTags(devicePath.getFullPath()); + + deviceIDS.add(deviceID); + + for (String tagkey : tagsMap.keySet()) { + String tagValue = tagsMap.get(tagkey); + Map> tagkeyMap = + tagInvertedIndex.computeIfAbsent(tagkey, key -> new HashMap<>()); + List ids = tagkeyMap.computeIfAbsent(tagValue, key -> new ArrayList<>()); + ids.add(deviceIDS.size() - 1); + } + } + + private void createTimeseries( + PartialPath path, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressor, + Map props) + throws MetadataException { + createTimeseries( + new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); + } + + private void createAlignedTimeSeries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries( + new CreateAlignedTimeSeriesPlan( + prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); + } + + @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { + PartialPath devicePath = plan.getPath().getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags) + "." + plan.getPath().getMeasurement()); + plan.setPath(path); + devicePath = plan.getPath().getDevicePath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { + throw new PathAlreadyExistException( + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } + } + idTable.createTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { + PartialPath devicePath = plan.getPrefixPath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags)); + plan.setPrefixPath(path); + devicePath = plan.getPrefixPath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (!deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", + devicePath.getFullPath()); + } else { + List measurements = plan.getMeasurements(); + List dataTypes = plan.getDataTypes(); + List encodings = plan.getEncodings(); + List compressors = plan.getCompressors(); + + List tmpMeasurements = new LinkedList<>(); + List tmpDataTypes = new LinkedList<>(); + List tmpEncodings = new LinkedList<>(); + List tmpCompressors = new LinkedList<>(); + for (int i = 0; i < measurements.size(); i++) { + String measurement = measurements.get(i); + if (!deviceEntry.getMeasurementMap().containsKey(measurement)) { + tmpMeasurements.add(measurements.get(i)); + tmpDataTypes.add(dataTypes.get(i)); + tmpEncodings.add(encodings.get(i)); + tmpCompressors.add(compressors.get(i)); + } + } + if (tmpMeasurements.size() == 0) + throw new PathAlreadyExistException(devicePath.getFullPath()); + plan.setMeasurements(tmpMeasurements); + plan.setDataTypes(tmpDataTypes); + plan.setEncodings(tmpEncodings); + plan.setCompressors(tmpCompressors); + } + } + idTable.createAlignedTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + return 0; + } + + @Override + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public List fetchSchemaBlackList(PathPatternTree patternTree) + throws MetadataException { + return null; + } + + @Override + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + int res = 0; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + for (IDeviceID deviceID : deviceIDs) { + res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); + } + return res; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS.size(); + } else { + return getDeviceIDsByInvertedIndex(pathPattern).size(); + } + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + Set res = new HashSet<>(); + String devicePath = pathPattern.getFullPath(); + if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry != null) { + res.add(pathPattern); + } + return res; + } + for (IDeviceID deviceID : deviceIDs) { + if (deviceID instanceof SHA256DeviceID) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map map = deviceEntry.getMeasurementMap(); + for (String m : map.keySet()) { + SchemaEntry schemaEntry = map.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new PartialPath( + diskSchemaEntry.seriesKey.substring( + 0, + diskSchemaEntry.seriesKey.length() + - diskSchemaEntry.measurementName.length() + - 1))); + break; + } + } else { + res.add(new PartialPath(deviceID.toStringID())); + } + } + return res; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select + // .. groupby level] + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + PartialPath devicePath = pathPattern.getDevicePath(); + // 批量查询.路径以".**"结尾,如： + // root.sg.tag1.a.** + // root.sg.tagx.c.tag2.v.** + // 点查询.路径不以".**",直接走IDTable,精确查询 + if (devicePath.getFullPath().endsWith(TAIL)) { + return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); + } else { + return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); + } + } + + private List getMeasurementPathsWithPointQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + String path = devicePath.getFullPath(); + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry == null) return res; + Map schemaMap = deviceEntry.getMeasurementMap(); + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + path, + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + + return res; + } + + private List getMeasurementPathsWithBatchQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); + for (IDeviceID deviceID : deviceIDs) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + MeasurementPath measurementPath = + new MeasurementPath( + new PartialPath(diskSchemaEntry.seriesKey), + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } else { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + deviceID.toStringID(), + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } + } + return res; + } + + // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + List res = getMeasurementPaths(pathPattern, isPrefixMatch); + Pair, Integer> result = new Pair<>(res, 0); + return result; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + // show 时间序列 + @Override // [iotdb/show timeseries] + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + List res = new ArrayList<>(); + Pair, Integer> result = new Pair<>(res, 0); + String path = plan.getPath().getFullPath(); + if (!path.endsWith(TAIL)) { + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry != null) { + Map measurementMap = deviceEntry.getMeasurementMap(); + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + path + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + return result; + } + List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); + for (IDeviceID deviceID : deviceIDs) { + getTimeSeriesResultOfDeviceFromIDTable(res, deviceID); + } + return result; + } + + private List getDeviceIdFromInvertedIndex(PartialPath devicePath) + throws MetadataException { + String path = devicePath.getFullPath(); + if (path.endsWith(TAIL)) { + path = path.substring(0, path.length() - TAIL.length()); + devicePath = new PartialPath(path); + } + if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS; + } else { + List res = new LinkedList<>(); + List ids = getDeviceIDsByInvertedIndex(devicePath); + if (ids.size() > 0) { + for (int id : ids) { + res.add(deviceIDS.get(id)); + } + } + return res; + } + } + + private void getTimeSeriesResultOfDeviceFromIDTable( + List res, IDeviceID deviceID) { + Map measurementMap = + idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new ShowTimeSeriesResult( + diskSchemaEntry.seriesKey, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } else { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + deviceID.toStringID() + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + } + + private List getDeviceIDsByInvertedIndex(PartialPath path) { + Map tags = pathToTags(path.getFullPath()); + List idsCollection = new ArrayList<>(tags.keySet().size()); + for (String tagKey : tags.keySet()) { + if (!tagInvertedIndex.containsKey(tagKey) + || !tagInvertedIndex.get(tagKey).containsKey(tags.get(tagKey))) { + return new ArrayList<>(); + } + List ids = tagInvertedIndex.get(tagKey).get(tags.get(tagKey)); + idsCollection.add(new ArrayList(ids)); + } + if (idsCollection.size() == 0) return new ArrayList<>(); + List ids = idsCollection.get(0); + for (int i = 1; i < idsCollection.size(); i++) { + List list = idsCollection.get(i); + ids.retainAll(list); + } + return ids; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + throw new UnsupportedOperationException(""); + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); + if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); + return new EntityMNode(storageGroupMNode, path.getFullPath()); + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + // insert data + @Override // [iotdb/insert ] + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + PartialPath devicePath = plan.getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + devicePath = new PartialPath(tagsToPath(tags)); + plan.setDevicePath(devicePath); + String[] measurementList = plan.getMeasurements(); + IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); + checkAlignedAndAutoCreateSeries(plan); + IMNode deviceMNode = getDeviceNode(devicePath); + IMeasurementMNode measurementMNode; + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (int i = 0; i < measurementList.length; i++) { + SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); + // measurementMNode = + // new MeasurementMNode( + // deviceMNode, + // measurementList[i], + // new MeasurementSchema( + // measurementList[i], + // schemaEntry.getTSDataType(), + // schemaEntry.getTSEncoding(), + // schemaEntry.getCompressionType()), + // null); + measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); + // check type is match + try { + SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); + } catch (DataTypeMismatchException mismatchException) { + if (!config.isEnablePartialInsert()) { + throw mismatchException; + } else { + // mark failed measurement + plan.markFailedMeasurementInsertion(i, mismatchException); + continue; + } + } + measurementMNodes[i] = measurementMNode; + } + plan.setDeviceID(deviceEntry.getDeviceID()); + plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); + return deviceMNode; + } + + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(getDataType.apply(i))); + + } else { + internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { + String[] measurementList = plan.getMeasurements(); + try { + if (plan.isAligned()) { + internalAlignedCreateTimeseries( + plan.getDevicePath(), + Arrays.asList(measurementList), + Arrays.asList(plan.getDataTypes())); + } else { + internalCreateTimeseries( + plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); + } + } catch (MetadataException e) { + if (!(e instanceof PathAlreadyExistException)) { + throw e; + } + } + } + + private void internalCreateTimeseries(PartialPath path, TSDataType dataType) + throws MetadataException { + createTimeseries( + path, + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor(), + Collections.emptyMap()); + } + + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, List measurements, List dataTypes) + throws MetadataException { + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + for (TSDataType dataType : dataTypes) { + encodings.add(getDefaultEncoding(dataType)); + compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { + throw new UnsupportedOperationException(""); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java index ae5e7b97b5716..9974b982c11ae 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaEntityNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaInternalNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaMeasurementNode; @@ -78,13 +79,49 @@ public Pair, Integer> searchMeasurementPaths( @Override public Pair, Integer> searchMeasurementPaths(PartialPath pathPattern) { - SchemaTreeMeasurementVisitor visitor = - new SchemaTreeMeasurementVisitor( - root, - pathPattern, - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); + SchemaTreeMeasurementVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = + new SchemaTreeMeasurementVisitor( + root, + pathPattern, + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + break; + case Tag: + if (pathPattern.getFullPath().contains(".**")) { + String measurement = pathPattern.getMeasurement(); + visitor = + new SchemaTreeMeasurementVisitor( + root, + ALL_MATCH_PATTERN.concatNode(measurement), + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + } else { + visitor = + new SchemaTreeMeasurementVisitor( + root, + pathPattern, + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + } + break; + default: + visitor = + new SchemaTreeMeasurementVisitor( + root, + ALL_MATCH_PATTERN, + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + break; + } return new Pair<>(visitor.getAllResult(), visitor.getNextOffset()); } @@ -107,7 +144,20 @@ public List getMatchedDevices(PartialPath pathPattern, boolean @Override public List getMatchedDevices(PartialPath pathPattern) { - SchemaTreeDeviceVisitor visitor = new SchemaTreeDeviceVisitor(root, pathPattern, false); + SchemaTreeDeviceVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = new SchemaTreeDeviceVisitor(root, pathPattern, false); + break; + case Tag: + visitor = new SchemaTreeDeviceVisitor(root, ALL_MATCH_PATTERN, false); + break; + default: + visitor = new SchemaTreeDeviceVisitor(root, ALL_MATCH_PATTERN, false); + break; + } return visitor.getAllResult(); } From 4a8e2b3199482a3bc3cb32a46a81fb9f87f128c6 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Sun, 18 Sep 2022 16:55:04 +0800 Subject: [PATCH 03/53] spotless apply --- .../iotdb/db/protocol/influxdb/handler/TagQueryHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java index 0f48caa5409f3..5344bba68c90d 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.protocol.influxdb.handler; -import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; From 69a9414b0f40728199c811412c1257858f1811d6 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 12:50:09 +0800 Subject: [PATCH 04/53] lsm --- lsm/pom.xml | 30 +++ .../org/apache/iotdb/lsm/context/Context.java | 101 +++++++++ .../apache/iotdb/lsm/context/ContextType.java | 27 +++ .../iotdb/lsm/context/DeleteContext.java | 43 ++++ .../iotdb/lsm/context/FlushContext.java | 43 ++++ .../iotdb/lsm/context/InsertContext.java | 50 +++++ .../iotdb/lsm/context/QueryContext.java | 43 ++++ .../org/apache/iotdb/lsm/example/Main.java | 209 ++++++++++++++++++ .../apache/iotdb/lsm/example/MemChunk.java | 44 ++++ .../apache/iotdb/lsm/example/MemGroup.java | 45 ++++ .../apache/iotdb/lsm/example/MemTable.java | 45 ++++ .../iotdb/lsm/example/MemTableManager.java | 75 +++++++ .../lsm/levelProcess/BasicLevelProcess.java | 52 +++++ .../lsm/levelProcess/DeleteLevelProcess.java | 31 +++ .../lsm/levelProcess/FlushLevelProcess.java | 30 +++ .../lsm/levelProcess/InsertLevelProcess.java | 31 +++ .../iotdb/lsm/levelProcess/LevelProcess.java | 29 +++ .../lsm/levelProcess/QueryLevelProcess.java | 31 +++ .../iotdb/lsm/manager/BasicLsmManager.java | 46 ++++ .../apache/iotdb/lsm/manager/LsmManager.java | 30 +++ .../iotdb/lsm/strategy/AccessStrategy.java | 35 +++ .../iotdb/lsm/strategy/BFSAccessStrategy.java | 56 +++++ .../lsm/strategy/PostOrderAccessStrategy.java | 47 ++++ .../lsm/strategy/PreOrderAccessStrategy.java | 44 ++++ .../lsm/strategy/RBFSAccessStrategy.java | 74 +++++++ 25 files changed, 1291 insertions(+) create mode 100644 lsm/pom.xml create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java diff --git a/lsm/pom.xml b/lsm/pom.xml new file mode 100644 index 0000000000000..1bbf055742d06 --- /dev/null +++ b/lsm/pom.xml @@ -0,0 +1,30 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.14.0-SNAPSHOT + + 4.0.0 + lsm + + 8 + 8 + + diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java new file mode 100644 index 0000000000000..381586e7c8294 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.AccessStrategy; +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +public class Context { + + // 类型 + ContextType type; + + // 访问策略 + AccessStrategy accessStrategy; + + // 所处的树深度 + int level; + + // 多少个线程处理该节点的子节点 + int threadNums; + + // 返回值 + Object result; + + public Context() { + accessStrategy = new PreOrderAccessStrategy(); + type = ContextType.NONE; + level = 0; + threadNums = 1; + } + + public Context( + ContextType type, + AccessStrategy accessStrategy, + int level, + boolean sync, + int threadNums, + Object result) { + this.type = type; + this.accessStrategy = accessStrategy; + this.level = level; + this.threadNums = threadNums; + this.result = result; + } + + public void setLevel(int level) { + this.level = level; + } + + public int getLevel() { + return level; + } + + public ContextType getType() { + return type; + } + + public int getThreadNums() { + return threadNums; + } + + public void setType(ContextType type) { + this.type = type; + } + + public void setThreadNums(int threadNums) { + this.threadNums = threadNums; + } + + public Object getResult() { + return result; + } + + public void setResult(Object result) { + this.result = result; + } + + public AccessStrategy getAccessStrategy() { + return accessStrategy; + } + + public void setAccessStrategy(AccessStrategy accessStrategy) { + this.accessStrategy = accessStrategy; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java new file mode 100644 index 0000000000000..3ff2e3509afb1 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +public enum ContextType { + NONE, + INSERT, + QUERY, + DELETE, + FLUSH; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java new file mode 100644 index 0000000000000..e00db284e7bce --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DeleteContext extends Context { + + List keys; + + public DeleteContext(Object... ks) { + super(); + keys = new ArrayList<>(); + keys.addAll(Arrays.asList(ks)); + type = ContextType.DELETE; + } + + public Object getKey() { + return keys.get(level); + } + + public int size() { + return keys.size(); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java new file mode 100644 index 0000000000000..ceacd2f636e5f --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.BFSAccessStrategy; +import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; + +public class FlushContext extends Context { + + // 最小的已经flush的层级 + int minimumFlushedLevel; + + public FlushContext() { + super(); + type = ContextType.FLUSH; + accessStrategy = new RBFSAccessStrategy(); + minimumFlushedLevel = Integer.MAX_VALUE; + } + + public int getMinimumFlushedLevel() { + return minimumFlushedLevel; + } + + public void setMinimumFlushedLevel(int minimumFlushedLevel) { + this.minimumFlushedLevel = minimumFlushedLevel; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java new file mode 100644 index 0000000000000..a34c9d5814ce8 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class InsertContext extends Context { + + List keys; + + Object value; + + public InsertContext(Object value, Object... keys) { + super(); + this.value = value; + this.keys = new ArrayList<>(); + this.keys.addAll(Arrays.asList(keys)); + level = 0; + type = ContextType.INSERT; + accessStrategy = new PreOrderAccessStrategy(); + } + + public Object getKey() { + return keys.get(level); + } + + public Object getValue() { + return value; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java new file mode 100644 index 0000000000000..9b8d440106b48 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class QueryContext extends Context { + + List keys; + + public QueryContext(Object... ks) { + super(); + keys = new ArrayList<>(); + keys.addAll(Arrays.asList(ks)); + type = ContextType.QUERY; + } + + public Object getKey() { + return keys.get(level); + } + + public int size() { + return keys.size(); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java new file mode 100644 index 0000000000000..015963e8af134 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import org.apache.iotdb.lsm.context.FlushContext; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.FlushLevelProcess; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Main { + public static void main(String[] args) { + MemTableManager memTableManager = new MemTableManager(); + System.out.println("-------------insert--------------"); + insertionExample(memTableManager); + System.out.println("-------------flush--------------"); + flushExample(memTableManager); + } + + public static void insertionExample(MemTableManager memTableManager) { + + BasicLsmManager baseLsmManager = + new BasicLsmManager().manager(memTableManager); + baseLsmManager + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemTableManager memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + List children = new ArrayList<>(); + if (deviceID / 65536 == maxDeviceID / 65536) { + children.add(memNode.getWorking()); + } else { + children.add(memNode.getImmutables().get(deviceID / 65536)); + } + return children; + } + + @Override + public void insert(MemTableManager memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + if (deviceID / 65536 == maxDeviceID / 65536) { + if (memNode.getWorking() == null) { + memNode.setWorking(new MemTable()); + } + } else if (deviceID > maxDeviceID) { + memNode + .getImmutables() + .put(memNode.getMaxDeviceID() / 65536, memNode.getWorking()); + memNode.setWorking(new MemTable()); + } + if (deviceID > maxDeviceID) { + memNode.setMaxDeviceID(deviceID); + } + } + }) + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemTable memNode, InsertContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemTable memNode, InsertContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemGroup()); + } + }) + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemGroup memNode, InsertContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemGroup memNode, InsertContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemChunk()); + } + }) + .nextLevel( + new InsertLevelProcess() { + @Override + public List getChildren(MemChunk memNode, InsertContext context) { + return null; + } + + @Override + public void insert(MemChunk memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + List deviceIDs = memNode.getDeviceIDS(); + deviceIDs.add(deviceID); + } + }); + + baseLsmManager.process(new InsertContext(1, null, "a", "b")); + baseLsmManager.process(new InsertContext(2, null, "a", "d")); + baseLsmManager.process(new InsertContext(3, null, "a", "e")); + baseLsmManager.process(new InsertContext(4, null, "a", "b")); + baseLsmManager.process(new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(new InsertContext(6, null, "a2", "b")); + baseLsmManager.process(new InsertContext(65535, null, "a", "b")); + baseLsmManager.process(new InsertContext(65536, null, "a", "b")); + baseLsmManager.process(new InsertContext(2, null, "a", "d")); + baseLsmManager.process(new InsertContext(3, null, "a", "e")); + baseLsmManager.process(new InsertContext(4, null, "a", "b")); + baseLsmManager.process(new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(new InsertContext(6, null, "a2", "b")); + System.out.println(memTableManager); + } + + public static void flushExample(MemTableManager memTableManager) { + BasicLsmManager flushManager = + new BasicLsmManager().manager(memTableManager); + + flushManager + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemTableManager memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemTableManager memNode, FlushContext context) { + List memTables = new ArrayList<>(); + memTables.addAll(memNode.getImmutables().values()); + if (memNode.getWorking() != null) memTables.add(memNode.getWorking()); + return memTables; + } + }) + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemTable memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemTable memNode, FlushContext context) { + List memGroups = new ArrayList<>(); + memGroups.addAll(memNode.getMap().values()); + return memGroups; + } + }) + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemGroup memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemGroup memNode, FlushContext context) { + List memChunk = new ArrayList<>(); + memChunk.addAll(memNode.getMap().values()); + return memChunk; + } + }) + .nextLevel( + new FlushLevelProcess() { + @Override + public void flush(MemChunk memNode, FlushContext context) { + System.out.println(memNode); + } + + @Override + public List getChildren(MemChunk memNode, FlushContext context) { + return new ArrayList<>(); + } + }); + + flushManager.process(new FlushContext()); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java new file mode 100644 index 0000000000000..c6e97fd52d9d2 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.ArrayList; +import java.util.List; + +// device list +public class MemChunk { + List deviceIDS; + + public MemChunk() { + deviceIDS = new ArrayList<>(); + } + + public List getDeviceIDS() { + return deviceIDS; + } + + public void setDeviceIDS(List deviceIDS) { + this.deviceIDS = deviceIDS; + } + + @Override + public String toString() { + return "MemChunk{" + deviceIDS.toString() + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java new file mode 100644 index 0000000000000..69f61d5d451fa --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.HashMap; +import java.util.Map; + +// tagvalue -> memChunk +public class MemGroup { + + Map map; + + public MemGroup() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemGroup{" + map.toString() + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java new file mode 100644 index 0000000000000..7cc82f19ecab7 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.HashMap; +import java.util.Map; + +// tagkey - > MemGroup +public class MemTable { + + private Map map; + + public MemTable() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemTable{" + map.toString() + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java new file mode 100644 index 0000000000000..d0f50071747c1 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.example; + +import java.util.HashMap; +import java.util.Map; + +// 管理working memtable , immutable memtables,框架用户自定义 +public class MemTableManager { + + // 可写的memtable + private MemTable working; + + // 只读的memtables + private Map immutables; + + // 记录已插入的最大的deviceid + private int maxDeviceID; + + public MemTableManager() { + working = new MemTable(); + immutables = new HashMap<>(); + maxDeviceID = 0; + } + + public MemTable getWorking() { + return working; + } + + public void setWorking(MemTable working) { + this.working = working; + } + + public Map getImmutables() { + return immutables; + } + + public void setImmutables(Map immutables) { + this.immutables = immutables; + } + + public int getMaxDeviceID() { + return maxDeviceID; + } + + public void setMaxDeviceID(int maxDeviceID) { + this.maxDeviceID = maxDeviceID; + } + + @Override + public String toString() { + return "MemTableManager{" + + "working=" + + working.toString() + + ", immutables=" + + immutables.toString() + + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java new file mode 100644 index 0000000000000..9a6359eab1714 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.Context; + +import java.util.List; + +public abstract class BasicLevelProcess implements LevelProcess { + LevelProcess next; + + public abstract void handle(I memNode, C context); + + public abstract List getChildren(I memNode, C context); + + @Override + public LevelProcess nextLevel(LevelProcess next) { + this.next = next; + return next; + } + + @Override + public void process(I memNode, C context) { + int currentLevel = context.getLevel(); + context.getAccessStrategy().execute(this, memNode, context); + context.setLevel(currentLevel); + } + + public boolean hasNext() { + return next != null; + } + + public LevelProcess getNext() { + return next; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java new file mode 100644 index 0000000000000..f4a4c97cd0f5c --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.DeleteContext; + +public abstract class DeleteLevelProcess extends BasicLevelProcess { + + public abstract void delete(I memNode, DeleteContext context); + + @Override + public void handle(I memNode, DeleteContext context) { + delete(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java new file mode 100644 index 0000000000000..00363c7683b75 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.FlushContext; + +public abstract class FlushLevelProcess extends BasicLevelProcess { + + public abstract void flush(I memNode, FlushContext context); + + public void handle(I memNode, FlushContext context) { + flush(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java new file mode 100644 index 0000000000000..91f7e24c1f250 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.InsertContext; + +public abstract class InsertLevelProcess extends BasicLevelProcess { + + public abstract void insert(I memNode, InsertContext context); + + @Override + public void handle(I memNode, InsertContext context) { + insert(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java new file mode 100644 index 0000000000000..aa04c2bf172b6 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.Context; + +public interface LevelProcess { + LevelProcess nextLevel(LevelProcess next); + + void process(I memNode, C context); + + void handle(I memNode, C context); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java new file mode 100644 index 0000000000000..5b3fc0e9ebb32 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.QueryContext; + +public abstract class QueryLevelProcess extends BasicLevelProcess { + + public abstract void query(I memNode, QueryContext context); + + @Override + public void handle(I memNode, QueryContext context) { + query(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java new file mode 100644 index 0000000000000..3d33dd1712864 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.LevelProcess; + +public class BasicLsmManager implements LsmManager { + + T root; + + LevelProcess levelProcess; + + @Override + public BasicLsmManager manager(T memNode) { + root = memNode; + return this; + } + + @Override + public void process(C context) { + levelProcess.process(root, context); + } + + @Override + public LevelProcess nextLevel(LevelProcess levelProcess) { + this.levelProcess = levelProcess; + return levelProcess; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java new file mode 100644 index 0000000000000..3d60af6f69f8e --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.LevelProcess; + +public interface LsmManager { + LsmManager manager(T memNode); + + void process(C context); + + LevelProcess nextLevel(LevelProcess next); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java new file mode 100644 index 0000000000000..d7780a5019877 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +// 表示内存节点访问策略（先序，后序） +public interface AccessStrategy{ + + /** + * + * @param levelProcess 保存当前节点和子节点的处理方法 + * @param memNode 当前待处理的节点 + * @param context 上下文信息 + */ + void execute( + BasicLevelProcess levelProcess, I memNode, C context); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java new file mode 100644 index 0000000000000..cf794b7097e61 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +public class BFSAccessStrategy implements AccessStrategy { + + Queue sameLevelMemNodes; + + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + List children = new ArrayList<>(); + int currentLevel = context.getLevel(); + // 第一个使用bfs策略的节点 + if (sameLevelMemNodes == null) { + sameLevelMemNodes = new LinkedList<>(); + levelProcess.handle(memNode, context); + children = levelProcess.getChildren(memNode, context); + } else { + while (!sameLevelMemNodes.isEmpty()) { + I node = (I) sameLevelMemNodes.poll(); + levelProcess.handle(node, context); + children.addAll(levelProcess.getChildren(node, context)); + } + } + sameLevelMemNodes.addAll(children); + context.setLevel(currentLevel + 1); + if (levelProcess.hasNext() && !sameLevelMemNodes.isEmpty()) { + levelProcess.getNext().process(null, context); + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java new file mode 100644 index 0000000000000..3079041763701 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +public class PostOrderAccessStrategy implements AccessStrategy { + + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + int currentLevel = context.getLevel(); + AccessStrategy accessStrategy = context.getAccessStrategy(); + List children = levelProcess.getChildren(memNode, context); + // 处理子节点 + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + levelProcess.getNext().process(child, context); + } + } + + context.setLevel(currentLevel); + context.setAccessStrategy(accessStrategy); + // 处理该节点 + levelProcess.handle(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java new file mode 100644 index 0000000000000..11f8be0844744 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +public class PreOrderAccessStrategy implements AccessStrategy { + + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + int currentLevel = context.getLevel(); + // 处理该节点 + levelProcess.handle(memNode, context); + List children = levelProcess.getChildren(memNode, context); + + // 处理子节点 + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + levelProcess.getNext().process(child, context); + } + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java new file mode 100644 index 0000000000000..e6b315eb145af --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.FlushContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +public class RBFSAccessStrategy implements AccessStrategy{ + @Override + public void execute(BasicLevelProcess levelProcess, I memNode, C context) { + FlushContext flushContext = (FlushContext) context; + int currentLevel = context.getLevel(); + // 如果当前节点是最深的一层节点 + if(!levelProcess.hasNext()){ + flushContext.setMinimumFlushedLevel(currentLevel); + } + // 如果是根节点 + if(currentLevel == 0){ + while(flushContext.getMinimumFlushedLevel() != currentLevel){ + List children = levelProcess.getChildren(memNode, context); + for(O child : children){ + // 处理子节点 + flushContext.setLevel(currentLevel+1); + levelProcess.getNext().process(child,context); + flushContext.setLevel(currentLevel); + } + // 每次处理完-1 + flushContext.setMinimumFlushedLevel(flushContext.getMinimumFlushedLevel() - 1); + } + // 处理root节点 + levelProcess.handle(memNode, context); + return; + } + + // 后序遍历，处理level == minimumFlushedLevel的节点 + + // 已经处理过，直接return + if(currentLevel > flushContext.getMinimumFlushedLevel()) return; + + + // 处理子节点 + if(currentLevel == flushContext.getMinimumFlushedLevel()){ + levelProcess.handle(memNode, context); + return; + } + List children = levelProcess.getChildren(memNode,context); + for(O child : children){ + flushContext.setLevel(currentLevel+1); + levelProcess.getNext().process(child,context); + flushContext.setLevel(currentLevel); + } + + } + +} From d147f35eb90d09424311d4b7849cff36e6f2a2c5 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 14:14:14 +0800 Subject: [PATCH 05/53] parent pom add lsm --- lsm/pom.xml | 4 ++- .../iotdb/lsm/context/FlushContext.java | 1 - .../iotdb/lsm/strategy/AccessStrategy.java | 5 ++- .../lsm/strategy/RBFSAccessStrategy.java | 32 +++++++++---------- pom.xml | 1 + 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lsm/pom.xml b/lsm/pom.xml index 1bbf055742d06..d6ff4cf992ad9 100644 --- a/lsm/pom.xml +++ b/lsm/pom.xml @@ -20,9 +20,11 @@ iotdb-parent org.apache.iotdb 0.14.0-SNAPSHOT + ../pom.xml 4.0.0 - lsm + iotdb-lsm + IoTDB lsm 8 8 diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java index ceacd2f636e5f..cb8c938175811 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.lsm.context; -import org.apache.iotdb.lsm.strategy.BFSAccessStrategy; import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; public class FlushContext extends Context { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java index d7780a5019877..9ff3f0f0a03e3 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -22,11 +22,10 @@ import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; // 表示内存节点访问策略（先序，后序） -public interface AccessStrategy{ +public interface AccessStrategy { /** - * - * @param levelProcess 保存当前节点和子节点的处理方法 + * @param levelProcess 保存当前节点和子节点的处理方法 * @param memNode 当前待处理的节点 * @param context 上下文信息 */ diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index e6b315eb145af..d9f4fa19004c2 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -24,23 +24,24 @@ import java.util.List; -public class RBFSAccessStrategy implements AccessStrategy{ +public class RBFSAccessStrategy implements AccessStrategy { @Override - public void execute(BasicLevelProcess levelProcess, I memNode, C context) { + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { FlushContext flushContext = (FlushContext) context; int currentLevel = context.getLevel(); // 如果当前节点是最深的一层节点 - if(!levelProcess.hasNext()){ + if (!levelProcess.hasNext()) { flushContext.setMinimumFlushedLevel(currentLevel); } // 如果是根节点 - if(currentLevel == 0){ - while(flushContext.getMinimumFlushedLevel() != currentLevel){ + if (currentLevel == 0) { + while (flushContext.getMinimumFlushedLevel() != currentLevel) { List children = levelProcess.getChildren(memNode, context); - for(O child : children){ + for (O child : children) { // 处理子节点 - flushContext.setLevel(currentLevel+1); - levelProcess.getNext().process(child,context); + flushContext.setLevel(currentLevel + 1); + levelProcess.getNext().process(child, context); flushContext.setLevel(currentLevel); } // 每次处理完-1 @@ -54,21 +55,18 @@ public void execute(BasicLevelProcess levelPr // 后序遍历，处理level == minimumFlushedLevel的节点 // 已经处理过，直接return - if(currentLevel > flushContext.getMinimumFlushedLevel()) return; - + if (currentLevel > flushContext.getMinimumFlushedLevel()) return; // 处理子节点 - if(currentLevel == flushContext.getMinimumFlushedLevel()){ + if (currentLevel == flushContext.getMinimumFlushedLevel()) { levelProcess.handle(memNode, context); return; } - List children = levelProcess.getChildren(memNode,context); - for(O child : children){ - flushContext.setLevel(currentLevel+1); - levelProcess.getNext().process(child,context); + List children = levelProcess.getChildren(memNode, context); + for (O child : children) { + flushContext.setLevel(currentLevel + 1); + levelProcess.getNext().process(child, context); flushContext.setLevel(currentLevel); } - } - } diff --git a/pom.xml b/pom.xml index a7f5012ec1189..a6c5d56c99384 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,7 @@ trigger-api rewrite-tsfile-tool external-api + lsm From 27f12fb72462cd8a9d8c6854f1d5f3ff52ee9552 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 18:05:02 +0800 Subject: [PATCH 06/53] lsm preOrder level fix --- lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java | 8 ++++---- .../apache/iotdb/lsm/levelProcess/BasicLevelProcess.java | 2 -- .../apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java | 1 + 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java index 015963e8af134..21796877734da 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -152,7 +152,7 @@ public static void flushExample(MemTableManager memTableManager) { new FlushLevelProcess() { @Override public void flush(MemTableManager memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override @@ -167,7 +167,7 @@ public List getChildren(MemTableManager memNode, FlushContext context) new FlushLevelProcess() { @Override public void flush(MemTable memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override @@ -181,7 +181,7 @@ public List getChildren(MemTable memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemGroup memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override @@ -195,7 +195,7 @@ public List getChildren(MemGroup memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemChunk memNode, FlushContext context) { - System.out.println(memNode); + System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); } @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java index 9a6359eab1714..f4ced68ec5a08 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java @@ -37,9 +37,7 @@ public LevelProcess nextLevel(LevelProcess next) { @Override public void process(I memNode, C context) { - int currentLevel = context.getLevel(); context.getAccessStrategy().execute(this, memNode, context); - context.setLevel(currentLevel); } public boolean hasNext() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java index 11f8be0844744..2c0b1b7b03a6c 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -40,5 +40,6 @@ public void execute( levelProcess.getNext().process(child, context); } } + context.setLevel(currentLevel); } } From 0e6e452f12f5c8f62f9824ea24b6e79c400bc0f5 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 7 Sep 2022 21:32:39 +0800 Subject: [PATCH 07/53] lsm preOrder level fix --- .../java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index d9f4fa19004c2..1505361947793 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -52,8 +52,6 @@ public void execute( return; } - // 后序遍历，处理level == minimumFlushedLevel的节点 - // 已经处理过，直接return if (currentLevel > flushContext.getMinimumFlushedLevel()) return; From 3935efc6342d77a7fda7e86dd05c52d92fcdbba7 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 8 Sep 2022 19:29:37 +0800 Subject: [PATCH 08/53] lsm add levelUpperBound --- .../org/apache/iotdb/lsm/context/Context.java | 26 +++++++++---------- .../iotdb/lsm/context/FlushContext.java | 15 ----------- .../org/apache/iotdb/lsm/example/Main.java | 8 +++--- .../lsm/strategy/RBFSAccessStrategy.java | 23 +++++++--------- 4 files changed, 26 insertions(+), 46 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java index 381586e7c8294..305b31747f6c3 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -35,6 +35,9 @@ public class Context { // 多少个线程处理该节点的子节点 int threadNums; + // 上界,大于该值的层级不会被处理 + int levelUpperBound; + // 返回值 Object result; @@ -43,20 +46,7 @@ public Context() { type = ContextType.NONE; level = 0; threadNums = 1; - } - - public Context( - ContextType type, - AccessStrategy accessStrategy, - int level, - boolean sync, - int threadNums, - Object result) { - this.type = type; - this.accessStrategy = accessStrategy; - this.level = level; - this.threadNums = threadNums; - this.result = result; + levelUpperBound = Integer.MAX_VALUE; } public void setLevel(int level) { @@ -98,4 +88,12 @@ public AccessStrategy getAccessStrategy() { public void setAccessStrategy(AccessStrategy accessStrategy) { this.accessStrategy = accessStrategy; } + + public int getLevelUpperBound() { + return levelUpperBound; + } + + public void setLevelUpperBound(int levelUpperBound) { + this.levelUpperBound = levelUpperBound; + } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java index cb8c938175811..2fd266396c8f8 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java @@ -21,22 +21,7 @@ import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; public class FlushContext extends Context { - - // 最小的已经flush的层级 - int minimumFlushedLevel; - public FlushContext() { - super(); - type = ContextType.FLUSH; accessStrategy = new RBFSAccessStrategy(); - minimumFlushedLevel = Integer.MAX_VALUE; - } - - public int getMinimumFlushedLevel() { - return minimumFlushedLevel; - } - - public void setMinimumFlushedLevel(int minimumFlushedLevel) { - this.minimumFlushedLevel = minimumFlushedLevel; } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java index 21796877734da..bea5c80e15873 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -152,7 +152,7 @@ public static void flushExample(MemTableManager memTableManager) { new FlushLevelProcess() { @Override public void flush(MemTableManager memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override @@ -167,7 +167,7 @@ public List getChildren(MemTableManager memNode, FlushContext context) new FlushLevelProcess() { @Override public void flush(MemTable memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override @@ -181,7 +181,7 @@ public List getChildren(MemTable memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemGroup memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override @@ -195,7 +195,7 @@ public List getChildren(MemGroup memNode, FlushContext context) { new FlushLevelProcess() { @Override public void flush(MemChunk memNode, FlushContext context) { - System.out.println(memNode + "-->[ level:" + context.getLevel() + " ]"); + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); } @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index 1505361947793..62d62295786fc 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -19,7 +19,6 @@ package org.apache.iotdb.lsm.strategy; import org.apache.iotdb.lsm.context.Context; -import org.apache.iotdb.lsm.context.FlushContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.List; @@ -28,24 +27,22 @@ public class RBFSAccessStrategy implements AccessStrategy { @Override public void execute( BasicLevelProcess levelProcess, I memNode, C context) { - FlushContext flushContext = (FlushContext) context; int currentLevel = context.getLevel(); - // 如果当前节点是最深的一层节点 - if (!levelProcess.hasNext()) { - flushContext.setMinimumFlushedLevel(currentLevel); + if (Integer.MAX_VALUE == context.getLevelUpperBound() && !levelProcess.hasNext()) { + context.setLevelUpperBound(context.getLevel()); } // 如果是根节点 if (currentLevel == 0) { - while (flushContext.getMinimumFlushedLevel() != currentLevel) { + while (context.getLevelUpperBound() != currentLevel) { List children = levelProcess.getChildren(memNode, context); for (O child : children) { // 处理子节点 - flushContext.setLevel(currentLevel + 1); + context.setLevel(currentLevel + 1); levelProcess.getNext().process(child, context); - flushContext.setLevel(currentLevel); + context.setLevel(currentLevel); } // 每次处理完-1 - flushContext.setMinimumFlushedLevel(flushContext.getMinimumFlushedLevel() - 1); + context.setLevelUpperBound(context.getLevelUpperBound() - 1); } // 处理root节点 levelProcess.handle(memNode, context); @@ -53,18 +50,18 @@ public void execute( } // 已经处理过，直接return - if (currentLevel > flushContext.getMinimumFlushedLevel()) return; + if (currentLevel > context.getLevelUpperBound()) return; // 处理子节点 - if (currentLevel == flushContext.getMinimumFlushedLevel()) { + if (currentLevel == context.getLevelUpperBound()) { levelProcess.handle(memNode, context); return; } List children = levelProcess.getChildren(memNode, context); for (O child : children) { - flushContext.setLevel(currentLevel + 1); + context.setLevel(currentLevel + 1); levelProcess.getNext().process(child, context); - flushContext.setLevel(currentLevel); + context.setLevel(currentLevel); } } } From 35efe5529de5d71093e09aee8978b5ad7ca0849e Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 9 Sep 2022 19:50:47 +0800 Subject: [PATCH 09/53] add TagInvertedIndex --- .../tagIndex/ITagInvertedIndex.java | 31 +++++++++++++++++++ .../tagIndex/TagInvertedIndex.java | 23 ++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java create mode 100644 lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java new file mode 100644 index 0000000000000..a6798ab0096c3 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import java.io.File; +import java.util.List; +import java.util.Map; + +public interface ITagInvertedIndex { + void addTag(String tagKey,String tagValue,int id); + void addTags(Map tags,int id); + void removeTag(String tagKey,String tagValue,int id); + void removeTags(Map tags,int id); + List getMatchedIDs(); +} diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java new file mode 100644 index 0000000000000..cbfbb9ee58983 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +public class TagInvertedIndex { + +} From a157158ae054c8c34051b5b6986e66d85ab58d44 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 9 Sep 2022 19:52:08 +0800 Subject: [PATCH 10/53] add TagInvertedIndex --- .../tagIndex/ITagInvertedIndex.java | 15 +++++++----- .../tagIndex/TagInvertedIndex.java | 23 ------------------- 2 files changed, 9 insertions(+), 29 deletions(-) delete mode 100644 lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index a6798ab0096c3..3934a6b680d8a 100644 --- a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -18,14 +18,17 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; -import java.io.File; import java.util.List; import java.util.Map; public interface ITagInvertedIndex { - void addTag(String tagKey,String tagValue,int id); - void addTags(Map tags,int id); - void removeTag(String tagKey,String tagValue,int id); - void removeTags(Map tags,int id); - List getMatchedIDs(); + void addTag(String tagKey, String tagValue, int id); + + void addTags(Map tags, int id); + + void removeTag(String tagKey, String tagValue, int id); + + void removeTags(Map tags, int id); + + List getMatchedIDs(Map tags); } diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java deleted file mode 100644 index cbfbb9ee58983..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; - -public class TagInvertedIndex { - -} From f20a6cd0755e4b2ed8b6f472aa29fd805552d947 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Tue, 13 Sep 2022 15:30:30 +0800 Subject: [PATCH 11/53] impl TagInvertedIndex --- lsm/pom.xml | 4 - .../iotdb/lsm/context/DeleteContext.java | 12 +- .../iotdb/lsm/context/QueryContext.java | 3 + pom.xml | 1 + schema-engine-tag/pom.xml | 51 ++++++ .../tagIndex/ITagInvertedIndex.java | 4 - .../tagIndex/TagInvertedIndex.java | 153 ++++++++++++++++++ .../tagIndex/deletion/DeletionManager.java | 41 +++++ .../tagIndex/deletion/MemChunkDeletion.java | 38 +++++ .../deletion/MemChunkGroupDeletion.java | 47 ++++++ .../tagIndex/deletion/MemTableDeletion.java | 56 +++++++ .../tagIndex/insertion/InsertionManager.java | 42 +++++ .../insertion/MemChunkGroupInsertion.java | 44 +++++ .../tagIndex/insertion/MemChunkInsertion.java | 38 +++++ .../tagIndex/insertion/MemTableInsertion.java | 48 ++++++ .../tagIndex/memtable/MemChunk.java | 52 ++++++ .../tagIndex/memtable/MemChunkGroup.java | 53 ++++++ .../tagIndex/memtable/MemTable.java | 84 ++++++++++ .../tagIndex/query/MemChunkGroupQuery.java | 41 +++++ .../tagIndex/query/MemChunkQuery.java | 42 +++++ .../tagIndex/query/MemTableQuery.java | 54 +++++++ .../tagIndex/query/QueryManager.java | 41 +++++ .../tagIndex/TagInvertedIndexTest.java | 148 +++++++++++++++++ .../org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++ 24 files changed, 1098 insertions(+), 9 deletions(-) create mode 100644 schema-engine-tag/pom.xml rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java (90%) create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java create mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java diff --git a/lsm/pom.xml b/lsm/pom.xml index d6ff4cf992ad9..081c60b0abc73 100644 --- a/lsm/pom.xml +++ b/lsm/pom.xml @@ -25,8 +25,4 @@ 4.0.0 iotdb-lsm IoTDB lsm - - 8 - 8 - diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java index e00db284e7bce..46d78b9ed5131 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.lsm.context; +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -26,17 +28,25 @@ public class DeleteContext extends Context { List keys; - public DeleteContext(Object... ks) { + Object value; + + public DeleteContext(Object value, Object... ks) { super(); + this.value = value; keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); type = ContextType.DELETE; + accessStrategy = new PostOrderAccessStrategy(); } public Object getKey() { return keys.get(level); } + public Object getValue() { + return value; + } + public int size() { return keys.size(); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java index 9b8d440106b48..7221b643f7fac 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.lsm.context; +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -31,6 +33,7 @@ public QueryContext(Object... ks) { keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); type = ContextType.QUERY; + accessStrategy = new PostOrderAccessStrategy(); } public Object getKey() { diff --git a/pom.xml b/pom.xml index a6c5d56c99384..6bb952ace414d 100644 --- a/pom.xml +++ b/pom.xml @@ -122,6 +122,7 @@ rewrite-tsfile-tool external-api lsm + schema-engine-tag diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml new file mode 100644 index 0000000000000..e44c73f76f055 --- /dev/null +++ b/schema-engine-tag/pom.xml @@ -0,0 +1,51 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.14.0-SNAPSHOT + ../pom.xml + + 4.0.0 + schema-engine-tag + schema-engine-tag + + + + org.roaringbitmap + RoaringBitmap + 0.9.32 + + + + org.apache.iotdb + iotdb-lsm + ${project.version} + provided + + + org.apache.iotdb + iotdb-server + ${project.version} + + + + \ No newline at end of file diff --git a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java similarity index 90% rename from lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java rename to schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index 3934a6b680d8a..cf5e1a19c0ea7 100644 --- a/lsm/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -22,12 +22,8 @@ import java.util.Map; public interface ITagInvertedIndex { - void addTag(String tagKey, String tagValue, int id); - void addTags(Map tags, int id); - void removeTag(String tagKey, String tagValue, int id); - void removeTags(Map tags, int id); List getMatchedIDs(Map tags); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java new file mode 100644 index 0000000000000..89c3260ed0e55 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.QueryContext; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class TagInvertedIndex implements ITagInvertedIndex { + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private int numOfDeviceIdsInMemTable; + + private MemTable workingMemTable; + + private MemTable unsequenceMemTable; + + private Map immutableMemTables; + + private int maxDeviceID; + + public TagInvertedIndex() { + workingMemTable = new MemTable(MemTable.WORKING); + unsequenceMemTable = new MemTable(MemTable.UNSEQUENCE); + immutableMemTables = new HashMap<>(); + numOfDeviceIdsInMemTable = config.getNumOfDeviceIdsInMemTable(); + maxDeviceID = 0; + } + + @Override + public synchronized void addTags(Map tags, int id) { + MemTable memTable = null; + // 出现乱序 + if (id < maxDeviceID) { + memTable = unsequenceMemTable; + } else { + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); + } + memTable = workingMemTable; + maxDeviceID = id; + } + for (Map.Entry tag : tags.entrySet()) { + addTag(memTable, tag.getKey(), tag.getValue(), id); + } + } + + @Override + public synchronized void removeTags(Map tags, int id) { + List memTables = new ArrayList<>(); + // 出现乱序 + if (inWorkingMemTable(id)) { + memTables.add(workingMemTable); + } else { + memTables.add(unsequenceMemTable); + memTables.add(immutableMemTables.get(id / numOfDeviceIdsInMemTable)); + } + for (Map.Entry tag : tags.entrySet()) { + removeTag(memTables, tag.getKey(), tag.getValue(), id); + } + } + + @Override + public synchronized List getMatchedIDs(Map tags) { + List memTables = new ArrayList<>(); + memTables.add(workingMemTable); + memTables.add(unsequenceMemTable); + memTables.addAll(immutableMemTables.values()); + RoaringBitmap roaringBitmap = new RoaringBitmap(); + int i = 0; + for (Map.Entry tag : tags.entrySet()) { + RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); + if (i == 0) roaringBitmap = rb; + else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); + i++; + } + return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); + } + + @Override + public String toString() { + return "TagInvertedIndex{" + + "numOfDeviceIdsInMemTable=" + + numOfDeviceIdsInMemTable + + ", workingMemTable=" + + workingMemTable + + ", unsequenceMemTable=" + + unsequenceMemTable + + ", immutableMemTables=" + + immutableMemTables + + ", maxDeviceID=" + + maxDeviceID + + '}'; + } + + private synchronized boolean inWorkingMemTable(int id) { + return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; + } + + private void addTag(MemTable memTable, String tagKey, String tagValue, int id) { + InsertContext insertContext = new InsertContext(id, tagKey, tagValue); + InsertionManager.getInstance().manager(memTable).process(insertContext); + } + + private void removeTag(List memTables, String tagKey, String tagValue, int id) { + DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); + for (MemTable memTable : memTables) { + DeletionManager.getInstance().manager(memTable).process(deleteContext); + } + } + + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) { + QueryContext queryContext = new QueryContext(tagKey, tagValue); + for (MemTable memTable : memTables) { + QueryManager.getInstance().manager(memTable).process(queryContext); + } + return (RoaringBitmap) queryContext.getResult(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java new file mode 100644 index 0000000000000..4125a1097db18 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +public class DeletionManager extends BasicLsmManager { + private DeletionManager() { + this.nextLevel(new MemTableDeletion()) + .nextLevel(new MemChunkGroupDeletion()) + .nextLevel(new MemChunkDeletion()); + } + + public static DeletionManager getInstance() { + return DeletionManagerHolder.INSTANCE; + } + + private static class DeletionManagerHolder { + private static final DeletionManager INSTANCE = new DeletionManager(); + + private DeletionManagerHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java new file mode 100644 index 0000000000000..c95a617765b66 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.List; + +public class MemChunkDeletion extends DeleteLevelProcess { + @Override + public List getChildren(MemChunk memNode, DeleteContext context) { + return null; + } + + @Override + public void delete(MemChunk memNode, DeleteContext context) { + Integer deviceID = (Integer) context.getValue(); + memNode.remove(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java new file mode 100644 index 0000000000000..651553fd86032 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +public class MemChunkGroupDeletion extends DeleteLevelProcess { + @Override + public List getChildren(MemChunkGroup memNode, DeleteContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + @Override + public void delete(MemChunkGroup memNode, DeleteContext context) { + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child == null || child.isEmpty()) { + memNode.remove(tagValue); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java new file mode 100644 index 0000000000000..0a1e5b3d05715 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class MemTableDeletion extends DeleteLevelProcess { + @Override + public List getChildren(MemTable memNode, DeleteContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + @Override + public void delete(MemTable memNode, DeleteContext context) { + if (memNode.isImmutable()) { + Set deletionList = memNode.getDeletionList(); + if (!deletionList.contains(context.getValue())) { + deletionList.add((Integer) context.getValue()); + } + return; + } + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child == null || child.isEmpty()) { + memNode.remove(tagKey); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java new file mode 100644 index 0000000000000..3e6d71493bd30 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +public class InsertionManager extends BasicLsmManager { + + private InsertionManager() { + this.nextLevel(new MemTableInsertion()) + .nextLevel(new MemChunkGroupInsertion()) + .nextLevel(new MemChunkInsertion()); + } + + public static InsertionManager getInstance() { + return InsertionManagerHolder.INSTANCE; + } + + private static class InsertionManagerHolder { + private static final InsertionManager INSTANCE = new InsertionManager(); + + private InsertionManagerHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java new file mode 100644 index 0000000000000..843084223f4ea --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +public class MemChunkGroupInsertion extends InsertLevelProcess { + @Override + public List getChildren(MemChunkGroup memNode, InsertContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + @Override + public void insert(MemChunkGroup memNode, InsertContext context) { + String tagValue = (String) context.getKey(); + memNode.put(tagValue); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java new file mode 100644 index 0000000000000..5dec23dcb7d1b --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.List; + +public class MemChunkInsertion extends InsertLevelProcess { + @Override + public List getChildren(MemChunk memNode, InsertContext context) { + return null; + } + + @Override + public void insert(MemChunk memNode, InsertContext context) { + Integer deviceID = (Integer) context.getValue(); + memNode.put(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java new file mode 100644 index 0000000000000..ef88f0591bc03 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +// memtable的insert操作 +public class MemTableInsertion extends InsertLevelProcess { + + @Override + public List getChildren(MemTable memNode, InsertContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + @Override + public void insert(MemTable memNode, InsertContext context) { + if (memNode.isImmutable()) return; + String tagKey = (String) context.getKey(); + memNode.put(tagKey); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java new file mode 100644 index 0000000000000..b46a770ca97e3 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import org.roaringbitmap.RoaringBitmap; + +// 管理设备id集合 +public class MemChunk { + private RoaringBitmap roaringBitmap; + + public MemChunk() { + roaringBitmap = new RoaringBitmap(); + } + + public boolean isEmpty() { + if (roaringBitmap == null) return true; + return roaringBitmap.isEmpty(); + } + + @Override + public String toString() { + return roaringBitmap.toString(); + } + + public void put(int id) { + roaringBitmap.add(id); + } + + public void remove(int id) { + roaringBitmap.remove(id); + } + + public RoaringBitmap getRoaringBitmap() { + return this.roaringBitmap; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java new file mode 100644 index 0000000000000..f6765030183a5 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.Map; + +public class MemChunkGroup { + private Map memChunkMap; + + public MemChunkGroup() { + memChunkMap = new HashMap<>(); + } + + public void put(String tagValue) { + if (!memChunkMap.containsKey(tagValue)) { + memChunkMap.put(tagValue, new MemChunk()); + } + } + + @Override + public String toString() { + return memChunkMap.toString(); + } + + public MemChunk get(String tagValue) { + return memChunkMap.get(tagValue); + } + + public void remove(String tagValue) { + memChunkMap.remove(tagValue); + } + + public boolean isEmpty() { + return memChunkMap.isEmpty(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java new file mode 100644 index 0000000000000..af47581ac9c3d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class MemTable { + public static final String WORKING = "working"; + + public static final String IMMUTABLE = "immutable"; + + public static final String UNSEQUENCE = "unsequence"; + + private Map memChunkGroupMap; + + private String status; + + private Set deletionList; + + public MemTable(String status) { + memChunkGroupMap = new HashMap<>(); + this.status = status; + deletionList = new HashSet<>(); + } + + public void put(String tagKey) { + if (this.status.equals(IMMUTABLE)) return; + if (!memChunkGroupMap.containsKey(tagKey)) { + memChunkGroupMap.put(tagKey, new MemChunkGroup()); + } + } + + @Override + public String toString() { + return "MemTable{" + + "memChunkGroupMap=" + + memChunkGroupMap + + ", status='" + + status + + '\'' + + ", deletionList=" + + deletionList + + '}'; + } + + public MemChunkGroup get(String tagKey) { + return memChunkGroupMap.get(tagKey); + } + + public void remove(String tagKey) { + memChunkGroupMap.remove(tagKey); + } + + public boolean isImmutable() { + return status.equals(IMMUTABLE); + } + + public Set getDeletionList() { + return deletionList; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java new file mode 100644 index 0000000000000..bf9656e82205d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +public class MemChunkGroupQuery extends QueryLevelProcess { + @Override + public List getChildren(MemChunkGroup memNode, QueryContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + @Override + public void query(MemChunkGroup memNode, QueryContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java new file mode 100644 index 0000000000000..d67de88f62bb4 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.List; + +public class MemChunkQuery extends QueryLevelProcess { + @Override + public List getChildren(MemChunk memNode, QueryContext context) { + return null; + } + + @Override + public void query(MemChunk memNode, QueryContext context) { + RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); + if (roaringBitmap == null) roaringBitmap = new RoaringBitmap(); + RoaringBitmap now = RoaringBitmap.or(roaringBitmap, memNode.getRoaringBitmap()); + context.setResult(now); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java new file mode 100644 index 0000000000000..b1acbf845b14e --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class MemTableQuery extends QueryLevelProcess { + + @Override + public List getChildren(MemTable memNode, QueryContext context) { + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + @Override + public void query(MemTable memNode, QueryContext context) { + // 如果是immutable，则需要在查询结果中删除deletionList中的id + if (memNode.isImmutable()) { + RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); + Set deletionList = memNode.getDeletionList(); + for (Integer id : deletionList) { + roaringBitmap.remove(id); + } + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java new file mode 100644 index 0000000000000..8987c49801f36 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +public class QueryManager extends BasicLsmManager { + private QueryManager() { + this.nextLevel(new MemTableQuery()) + .nextLevel(new MemChunkGroupQuery()) + .nextLevel(new MemChunkQuery()); + } + + public static QueryManager getInstance() { + return QueryManagerHolder.INSTANCE; + } + + private static class QueryManagerHolder { + private static final QueryManager INSTANCE = new QueryManager(); + + private QueryManagerHolder() {} + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java new file mode 100644 index 0000000000000..4a17213c77ba8 --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.tsfile.utils.Pair; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class TagInvertedIndexTest { + private String[][] record = + new String[][] { + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + }; + + private int numOfDeviceIdsInMemTable; + + private TagInvertedIndex tagInvertedIndex; + + @Before + public void setUp() throws Exception { + numOfDeviceIdsInMemTable = + IoTDBDescriptor.getInstance().getConfig().getNumOfDeviceIdsInMemTable(); + IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(3); + tagInvertedIndex = new TagInvertedIndex(); + } + + @After + public void tearDown() throws Exception { + IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + tagInvertedIndex = null; + } + + public void addTags() { + List, Integer>> records = generateTags(); + for (Pair, Integer> pair : records) { + tagInvertedIndex.addTags(pair.left, pair.right); + } + } + + public void removeTags() { + Pair, Integer> tags = generateTag(record[0]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[1]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[6]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[13]); + tagInvertedIndex.removeTags(tags.left, tags.right); + } + + @Test + public void getMatchedIDs() { + addTags(); + System.out.println("------------addTags------------"); + System.out.println(tagInvertedIndex); + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); + + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(1, 2, 3, 4, 5, 7, 12, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(1, 3, 12, 15, 16); + assertEquals(verify, ids); + + removeTags(); + System.out.println("------------removeTags------------"); + System.out.println(tagInvertedIndex); + ids = tagInvertedIndex.getMatchedIDs(tags1); + verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); + + System.out.println("------------query------------"); + System.out.println(tagInvertedIndex); + } + + private List, Integer>> generateTags() { + List, Integer>> pairs = new ArrayList<>(); + for (String[] strings : record) { + pairs.add(generateTag(strings)); + } + return pairs; + } + + private Pair, Integer> generateTag(String[] strings) { + Map tags = new HashMap<>(); + int i = 0; + for (; i < strings.length - 1; i++) { + String[] str = strings[i].split("="); + tags.put(str[0], str[1]); + } + Pair, Integer> pair = new Pair<>(tags, Integer.valueOf(strings[i])); + return pair; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 50d69dc5aa53a..e591fe56aa09b 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -866,6 +866,8 @@ public class IoTDBConfig { */ private boolean enableIDTableLogFile = false; + private int numOfDeviceIdsInMemTable = 65536; + /** whether to use persistent schema mode */ private String schemaEngineMode = "Memory"; @@ -2867,6 +2869,14 @@ public void setEnableIDTableLogFile(boolean enableIDTableLogFile) { this.enableIDTableLogFile = enableIDTableLogFile; } + public int getNumOfDeviceIdsInMemTable() { + return numOfDeviceIdsInMemTable; + } + + public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { + this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; + } + public String getSchemaEngineMode() { return schemaEngineMode; } From 3164fd581fec8e9134da1f3705406e49950fa984 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Tue, 13 Sep 2022 18:58:32 +0800 Subject: [PATCH 12/53] impl TagInvertedIndex --- schema-engine-tag/pom.xml | 11 +- .../tagSchemaRegion/MockTagSchemaRegion.java | 325 ++++++++++++++++++ .../tagSchemaRegion/TagSchemaRegion.java | 21 ++ .../tagIndex/TagInvertedIndexTest.java | 40 +-- 4 files changed, 368 insertions(+), 29 deletions(-) create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml index e44c73f76f055..c5b8f565525a0 100644 --- a/schema-engine-tag/pom.xml +++ b/schema-engine-tag/pom.xml @@ -15,31 +15,25 @@ See the License for the specific language governing permissions and limitations under the License. --> - + iotdb-parent org.apache.iotdb 0.14.0-SNAPSHOT - ../pom.xml 4.0.0 schema-engine-tag schema-engine-tag - org.roaringbitmap RoaringBitmap 0.9.32 - org.apache.iotdb iotdb-lsm ${project.version} - provided org.apache.iotdb @@ -47,5 +41,4 @@ ${project.version} - - \ No newline at end of file + diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java new file mode 100644 index 0000000000000..184ce789b99cd --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.Pair; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +public class MockTagSchemaRegion implements ISchemaRegion { + + @Override + public void init() throws MetadataException {} + + @Override + public void clear() {} + + @Override + public void forceMlog() {} + + @Override + public SchemaRegionId getSchemaRegionId() { + return null; + } + + @Override + public String getStorageGroupFullPath() { + return null; + } + + @Override + public void deleteSchemaRegion() throws MetadataException {} + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) {} + + @Override + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {} + + @Override + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {} + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {} + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + return false; + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + return null; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + return null; + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + return null; + } + + @Override + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + return null; + } + + @Override + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + @Override + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + return null; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + return null; + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + return null; + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {} + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + return null; + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + return null; + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + return false; + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {} + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {} + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {} + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java new file mode 100644 index 0000000000000..2f04d7c095a73 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +public class TagSchemaRegion {} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index 4a17213c77ba8..a927c2ebbce79 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -36,24 +36,24 @@ public class TagInvertedIndexTest { private String[][] record = new String[][] { - {"tag1=q", "tag2=a", "1"}, - {"tag1=q", "tag2=s", "2"}, - {"tag1=q", "tag2=a", "tag3=z", "3"}, - {"tag1=q", "tag2=s", "5"}, - {"tag1=w", "tag2=d", "6"}, - {"tag1=q", "tag2=d", "tag3=e", "7"}, - {"tag1=q", "tag3=v", "4"}, - {"tag1=r", "tag2=d", "9"}, - {"tag1=t", "tag2=f", "10"}, - {"tag1=t", "tag2=g", "8"}, - {"tag1=t", "tag2=h", "11"}, - {"tag1=y", "tag2=j", "13"}, - {"tag1=u", "tag2=k", "14"}, - {"tag1=q", "tag2=a", "tag3=l", "12"}, - {"tag1=q", "tag2=a", "tag3=x", "15"}, - {"tag1=q", "tag2=b", "tag3=x", "18"}, - {"tag1=y", "tag2=a", "tag4=z", "17"}, - {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, }; private int numOfDeviceIdsInMemTable; @@ -86,9 +86,9 @@ public void removeTags() { tagInvertedIndex.removeTags(tags.left, tags.right); tags = generateTag(record[1]); tagInvertedIndex.removeTags(tags.left, tags.right); - tags = generateTag(record[6]); + tags = generateTag(record[3]); tagInvertedIndex.removeTags(tags.left, tags.right); - tags = generateTag(record[13]); + tags = generateTag(record[11]); tagInvertedIndex.removeTags(tags.left, tags.right); } From 793e7c849347a5d83f271031612ec0e3d490d4be Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 16:40:41 +0800 Subject: [PATCH 13/53] impl wal --- .../iotdb/lsm/context/DeleteContext.java | 9 ++ .../iotdb/lsm/context/InsertContext.java | 9 ++ .../org/apache/iotdb/lsm/example/Main.java | 38 +++--- .../iotdb/lsm/manager/BasicLsmManager.java | 16 +-- .../apache/iotdb/lsm/manager/LsmManager.java | 3 +- .../org/apache/iotdb/lsm/wal/IWALReader.java | 31 +++++ .../org/apache/iotdb/lsm/wal/IWALWriter.java | 30 +++++ .../org/apache/iotdb/lsm/wal/WALReader.java | 92 ++++++++++++++ .../org/apache/iotdb/lsm/wal/WALRecord.java | 38 ++++++ .../org/apache/iotdb/lsm/wal/WALWriter.java | 98 +++++++++++++++ .../tagIndex/TagInvertedIndex.java | 105 ++++++++++------ .../tagIndex/deletion/DeletionManager.java | 23 ++-- .../tagIndex/insertion/InsertionManager.java | 26 ++-- .../tagIndex/memtable/MemTable.java | 2 - .../tagIndex/query/QueryManager.java | 17 +-- .../tagIndex/recover/RecoverManager.java | 33 +++++ .../tagIndex/wal/WALEntry.java | 95 ++++++++++++++ .../tagIndex/wal/WALManager.java | 117 ++++++++++++++++++ .../tagIndex/TagInvertedIndexTest.java | 53 +++++--- 19 files changed, 717 insertions(+), 118 deletions(-) create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java create mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java index 46d78b9ed5131..2b99196b0c3aa 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -39,10 +39,19 @@ public DeleteContext(Object value, Object... ks) { accessStrategy = new PostOrderAccessStrategy(); } + public DeleteContext(Object value,List keys){ + this.value = value; + this.keys = keys; + } + public Object getKey() { return keys.get(level); } + public List getKeys() { + return keys; + } + public Object getValue() { return value; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java index a34c9d5814ce8..e8d00da4979d7 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java @@ -40,10 +40,19 @@ public InsertContext(Object value, Object... keys) { accessStrategy = new PreOrderAccessStrategy(); } + public InsertContext(Object value,List keys){ + this.value = value; + this.keys = keys; + } + public Object getKey() { return keys.get(level); } + public List getKeys() { + return keys; + } + public Object getValue() { return value; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java index bea5c80e15873..7dbe793ac5fcd 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java @@ -29,7 +29,7 @@ import java.util.Map; public class Main { - public static void main(String[] args) { + public static void main(String[] args) throws Exception { MemTableManager memTableManager = new MemTableManager(); System.out.println("-------------insert--------------"); insertionExample(memTableManager); @@ -37,10 +37,10 @@ public static void main(String[] args) { flushExample(memTableManager); } - public static void insertionExample(MemTableManager memTableManager) { + public static void insertionExample(MemTableManager memTableManager) throws Exception { BasicLsmManager baseLsmManager = - new BasicLsmManager().manager(memTableManager); + new BasicLsmManager(); baseLsmManager .nextLevel( new InsertLevelProcess() { @@ -127,25 +127,25 @@ public void insert(MemChunk memNode, InsertContext context) { } }); - baseLsmManager.process(new InsertContext(1, null, "a", "b")); - baseLsmManager.process(new InsertContext(2, null, "a", "d")); - baseLsmManager.process(new InsertContext(3, null, "a", "e")); - baseLsmManager.process(new InsertContext(4, null, "a", "b")); - baseLsmManager.process(new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(new InsertContext(6, null, "a2", "b")); - baseLsmManager.process(new InsertContext(65535, null, "a", "b")); - baseLsmManager.process(new InsertContext(65536, null, "a", "b")); - baseLsmManager.process(new InsertContext(2, null, "a", "d")); - baseLsmManager.process(new InsertContext(3, null, "a", "e")); - baseLsmManager.process(new InsertContext(4, null, "a", "b")); - baseLsmManager.process(new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(new InsertContext(6, null, "a2", "b")); + baseLsmManager.process(memTableManager, new InsertContext(1, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); + baseLsmManager.process(memTableManager, new InsertContext(65535, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(65536, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); System.out.println(memTableManager); } - public static void flushExample(MemTableManager memTableManager) { + public static void flushExample(MemTableManager memTableManager) throws Exception { BasicLsmManager flushManager = - new BasicLsmManager().manager(memTableManager); + new BasicLsmManager(); flushManager .nextLevel( @@ -204,6 +204,6 @@ public List getChildren(MemChunk memNode, FlushContext context) { } }); - flushManager.process(new FlushContext()); + flushManager.process(memTableManager, new FlushContext()); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java index 3d33dd1712864..a5b91439662aa 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -21,21 +21,21 @@ import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -public class BasicLsmManager implements LsmManager { +import java.io.IOException; - T root; +public class BasicLsmManager implements LsmManager { LevelProcess levelProcess; - @Override - public BasicLsmManager manager(T memNode) { - root = memNode; - return this; - } + public void preProcess(T root, C context) throws Exception {} + + public void postProcess(T root, C context) throws Exception{} @Override - public void process(C context) { + public void process(T root, C context) throws Exception{ + preProcess(root, context); levelProcess.process(root, context); + postProcess(root, context); } @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java index 3d60af6f69f8e..2940ad60b1751 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java @@ -22,9 +22,8 @@ import org.apache.iotdb.lsm.levelProcess.LevelProcess; public interface LsmManager { - LsmManager manager(T memNode); - void process(C context); + void process(T memNode, C context) throws Exception; LevelProcess nextLevel(LevelProcess next); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java new file mode 100644 index 0000000000000..6d52ef1577fcd --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.FileNotFoundException; +import java.io.IOException; + +public interface IWALReader { + + void close() throws IOException; + + boolean hasNext() throws FileNotFoundException; + + WALRecord next() throws FileNotFoundException; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java new file mode 100644 index 0000000000000..6d3fecb21e2eb --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.IOException; + +public interface IWALWriter { + + void write(WALRecord walRecord) throws IOException; + + void force() throws IOException; + + void close() throws IOException; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java new file mode 100644 index 0000000000000..a3fef56b0860d --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.NoSuchElementException; + +public class WALReader implements IWALReader { + private static final Logger logger = LoggerFactory.getLogger(WALReader.class); + private final File logFile; + private final WALRecord prototype; + private final DataInputStream logStream; + private WALRecord nextRecord; + private boolean fileCorrupted = false; + + public WALReader(File logFile, WALRecord prototype) throws IOException { + this.logFile = logFile; + this.logStream = + new DataInputStream(new BufferedInputStream(Files.newInputStream(logFile.toPath()))); + this.prototype = prototype; + } + + @Override + public void close() throws IOException { + logStream.close(); + } + + @Override + public boolean hasNext() { + if (nextRecord != null) { + return true; + } + try { + if (fileCorrupted) { + return false; + } + int logSize = logStream.readInt(); + if (logSize <= 0) { + return false; + } + nextRecord = prototype.clone(); + nextRecord.deserialize(logStream); + } catch (EOFException e) { + logger.info(""); + return false; + } catch (IOException e) { + logger.warn(""); + fileCorrupted = true; + return false; + } + return true; + } + + @Override + public WALRecord next() { + if (nextRecord == null) { + throw new NoSuchElementException(); + } + WALRecord walRecord = nextRecord; + nextRecord = null; + return walRecord; + } + + @Override + public String toString() { + return "WALReader{" + "logFile=" + logFile + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java new file mode 100644 index 0000000000000..a44b8f9d243c5 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public abstract class WALRecord implements Cloneable { + public abstract void serialize(ByteBuffer buffer); + + public abstract void deserialize(DataInputStream stream) throws IOException; + + @Override + public WALRecord clone() { + try { + return (WALRecord) super.clone(); + } catch (CloneNotSupportedException e) { + throw new AssertionError(); + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java new file mode 100644 index 0000000000000..dccd8a92233db --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.FileChannel; + +public class WALWriter implements IWALWriter { + private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); + private File logFile; + private FileOutputStream fileOutputStream; + private FileChannel channel; + private final ByteBuffer lengthBuffer = ByteBuffer.allocate(4); + private final ByteBuffer walBuffer = ByteBuffer.allocate(10000); + private final boolean forceEachWrite; + + public WALWriter(File logFile, boolean forceEachWrite) throws FileNotFoundException { + this.logFile = logFile; + this.forceEachWrite = forceEachWrite; + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + } + + @Override + public void write(WALRecord walRecord) throws IOException { + if (channel == null) { + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + } + walBuffer.clear(); + walRecord.serialize(walBuffer); + walBuffer.flip(); + int logSize = walBuffer.limit(); + lengthBuffer.clear(); + lengthBuffer.putInt(logSize); + lengthBuffer.flip(); + + try { + channel.write(lengthBuffer); + channel.write(walBuffer); + + if (this.forceEachWrite) { + channel.force(true); + } + } catch (ClosedChannelException ignored) { + logger.warn("someone interrupt current thread, so no need to do write for io safety"); + } + } + + @Override + public void force() throws IOException { + if (channel != null && channel.isOpen()) { + channel.force(true); + } + } + + @Override + public void close() throws IOException { + if (channel != null) { + if (channel.isOpen()) { + channel.force(true); + } + fileOutputStream.close(); + fileOutputStream = null; + channel.close(); + channel = null; + } + } + + @Override + public String toString() { + return "WALLogWriter{" + "logFile=" + logFile + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 89c3260ed0e55..71c2ee923106a 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -18,18 +18,25 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover.RecoverManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.wal.WALWriter; import org.roaringbitmap.RoaringBitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -38,44 +45,61 @@ import java.util.stream.Collectors; public class TagInvertedIndex implements ITagInvertedIndex { + private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private int numOfDeviceIdsInMemTable; + private final InsertionManager insertionManager; - private MemTable workingMemTable; + private final DeletionManager deletionManager; + + private final QueryManager queryManager; + + private final WALManager walManager; + + private final RecoverManager recoverManager; - private MemTable unsequenceMemTable; + private final int numOfDeviceIdsInMemTable; - private Map immutableMemTables; + private final Map immutableMemTables; + + private MemTable workingMemTable; private int maxDeviceID; - public TagInvertedIndex() { + public TagInvertedIndex(String schemaDirPath) throws IOException { + walManager = new WALManager(schemaDirPath); + insertionManager = new InsertionManager(walManager); + deletionManager = new DeletionManager(walManager); + recoverManager = new RecoverManager(walManager); + queryManager = new QueryManager(); workingMemTable = new MemTable(MemTable.WORKING); - unsequenceMemTable = new MemTable(MemTable.UNSEQUENCE); immutableMemTables = new HashMap<>(); numOfDeviceIdsInMemTable = config.getNumOfDeviceIdsInMemTable(); maxDeviceID = 0; + recover(); + } + + public synchronized void recover(){ + recoverManager.recover(this); } @Override public synchronized void addTags(Map tags, int id) { MemTable memTable = null; - // 出现乱序 - if (id < maxDeviceID) { - memTable = unsequenceMemTable; - } else { - if (!inWorkingMemTable(id)) { - workingMemTable.setStatus(MemTable.IMMUTABLE); - immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); - workingMemTable = new MemTable(MemTable.WORKING); - } - memTable = workingMemTable; - maxDeviceID = id; + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); } - for (Map.Entry tag : tags.entrySet()) { - addTag(memTable, tag.getKey(), tag.getValue(), id); + memTable = workingMemTable; + maxDeviceID = id; + try { + for (Map.Entry tag : tags.entrySet()) { + addTag(memTable, tag.getKey(), tag.getValue(), id); + } + }catch (Exception e){ + logger.error(e.getMessage()); } } @@ -86,11 +110,14 @@ public synchronized void removeTags(Map tags, int id) { if (inWorkingMemTable(id)) { memTables.add(workingMemTable); } else { - memTables.add(unsequenceMemTable); memTables.add(immutableMemTables.get(id / numOfDeviceIdsInMemTable)); } - for (Map.Entry tag : tags.entrySet()) { - removeTag(memTables, tag.getKey(), tag.getValue(), id); + try { + for (Map.Entry tag : tags.entrySet()) { + removeTag(memTables, tag.getKey(), tag.getValue(), id); + } + }catch (Exception e){ + logger.error(e.getMessage()); } } @@ -98,15 +125,18 @@ public synchronized void removeTags(Map tags, int id) { public synchronized List getMatchedIDs(Map tags) { List memTables = new ArrayList<>(); memTables.add(workingMemTable); - memTables.add(unsequenceMemTable); memTables.addAll(immutableMemTables.values()); RoaringBitmap roaringBitmap = new RoaringBitmap(); int i = 0; - for (Map.Entry tag : tags.entrySet()) { - RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); - if (i == 0) roaringBitmap = rb; - else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); - i++; + try { + for (Map.Entry tag : tags.entrySet()) { + RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); + if (i == 0) roaringBitmap = rb; + else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); + i++; + } + }catch (Exception e){ + logger.error(e.getMessage()); } return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); } @@ -118,8 +148,6 @@ public String toString() { + numOfDeviceIdsInMemTable + ", workingMemTable=" + workingMemTable - + ", unsequenceMemTable=" - + unsequenceMemTable + ", immutableMemTables=" + immutableMemTables + ", maxDeviceID=" @@ -131,23 +159,28 @@ private synchronized boolean inWorkingMemTable(int id) { return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; } - private void addTag(MemTable memTable, String tagKey, String tagValue, int id) { + private void addTag(MemTable memTable, String tagKey, String tagValue, int id) throws Exception { InsertContext insertContext = new InsertContext(id, tagKey, tagValue); - InsertionManager.getInstance().manager(memTable).process(insertContext); + insertionManager.process(memTable, insertContext); } - private void removeTag(List memTables, String tagKey, String tagValue, int id) { + private void removeTag(List memTables, String tagKey, String tagValue, int id) throws Exception { DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); for (MemTable memTable : memTables) { - DeletionManager.getInstance().manager(memTable).process(deleteContext); + deletionManager.process(memTable, deleteContext); } } - private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) { + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) throws Exception { QueryContext queryContext = new QueryContext(tagKey, tagValue); for (MemTable memTable : memTables) { - QueryManager.getInstance().manager(memTable).process(queryContext); + queryManager.process(memTable, queryContext); } return (RoaringBitmap) queryContext.getResult(); } + + @TestOnly + public void clear() throws IOException { + walManager.close(); + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index 4125a1097db18..0d8e62a769f67 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -19,23 +19,24 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; +import java.io.IOException; + public class DeletionManager extends BasicLsmManager { - private DeletionManager() { - this.nextLevel(new MemTableDeletion()) - .nextLevel(new MemChunkGroupDeletion()) - .nextLevel(new MemChunkDeletion()); - } - public static DeletionManager getInstance() { - return DeletionManagerHolder.INSTANCE; - } + WALManager walManager; - private static class DeletionManagerHolder { - private static final DeletionManager INSTANCE = new DeletionManager(); + public DeletionManager(WALManager walManager) { + this.walManager = walManager; + initLevelProcess(); + } - private DeletionManagerHolder() {} + private void initLevelProcess() { + this.nextLevel(new MemTableDeletion()) + .nextLevel(new MemChunkGroupDeletion()) + .nextLevel(new MemChunkDeletion()); } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java index 3e6d71493bd30..4b4dd29ee8c62 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -19,24 +19,30 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; +import java.io.IOException; + public class InsertionManager extends BasicLsmManager { - private InsertionManager() { - this.nextLevel(new MemTableInsertion()) - .nextLevel(new MemChunkGroupInsertion()) - .nextLevel(new MemChunkInsertion()); - } + private WALManager walManager; - public static InsertionManager getInstance() { - return InsertionManagerHolder.INSTANCE; + public InsertionManager(WALManager walManager){ + this.walManager = walManager; + initLevelProcess(); } - private static class InsertionManagerHolder { - private static final InsertionManager INSTANCE = new InsertionManager(); + @Override + public void preProcess(MemTable root, InsertContext context) throws IOException { + walManager.write(context); + } - private InsertionManagerHolder() {} + private void initLevelProcess() { + this.nextLevel(new MemTableInsertion()) + .nextLevel(new MemChunkGroupInsertion()) + .nextLevel(new MemChunkInsertion()); } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java index af47581ac9c3d..f0730e162df1c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -28,8 +28,6 @@ public class MemTable { public static final String IMMUTABLE = "immutable"; - public static final String UNSEQUENCE = "unsequence"; - private Map memChunkGroupMap; private String status; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java index 8987c49801f36..4a8e3fdbcb0eb 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java @@ -23,19 +23,14 @@ import org.apache.iotdb.lsm.manager.BasicLsmManager; public class QueryManager extends BasicLsmManager { - private QueryManager() { - this.nextLevel(new MemTableQuery()) - .nextLevel(new MemChunkGroupQuery()) - .nextLevel(new MemChunkQuery()); - } - public static QueryManager getInstance() { - return QueryManagerHolder.INSTANCE; + public QueryManager() { + initLevelProcess(); } - private static class QueryManagerHolder { - private static final QueryManager INSTANCE = new QueryManager(); - - private QueryManagerHolder() {} + private void initLevelProcess() { + this.nextLevel(new MemTableQuery()) + .nextLevel(new MemChunkGroupQuery()) + .nextLevel(new MemChunkQuery()); } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java new file mode 100644 index 0000000000000..2c3d38bc06278 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; + +public class RecoverManager { + + WALManager walManager; + + public RecoverManager(WALManager walManager) { + this.walManager = walManager; + } + + public void recover(TagInvertedIndex tagInvertedIndex) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java new file mode 100644 index 0000000000000..e70ad62100b10 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.lsm.wal.WALRecord; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class WALEntry extends WALRecord { + private int type; + + private List keys; + + private int deviceID; + + public WALEntry() {} + + public WALEntry(int type, List keys, int deviceID) { + this.type = type; + this.keys = keys; + this.deviceID = deviceID; + } + + @Override + public void serialize(ByteBuffer buffer) { + ReadWriteIOUtils.write(type, buffer); + ReadWriteIOUtils.write(deviceID, buffer); + ReadWriteIOUtils.write(keys.size(), buffer); + for (String key : keys) { + ReadWriteIOUtils.write(key, buffer); + } + } + + @Override + public void deserialize(DataInputStream stream) throws IOException { + this.type = stream.readInt(); + this.deviceID = stream.readInt(); + int length = stream.readInt(); + this.keys = new ArrayList<>(); + for (int i = 0; i < length; i++) { + String key = ReadWriteIOUtils.readString(stream); + keys.add(key); + } + } + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + public List getKeys() { + return keys; + } + + public void setKeys(List keys) { + this.keys = keys; + } + + public int getDeviceID() { + return deviceID; + } + + public void setDeviceID(int deviceID) { + this.deviceID = deviceID; + } + + @Override + public String toString() { + return "WALEntry{" + "type=" + type + ", keys=" + keys + ", deviceID=" + deviceID + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java new file mode 100644 index 0000000000000..88bdf68dd3143 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.wal.WALReader; +import org.apache.iotdb.lsm.wal.WALWriter; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class WALManager { + private static final String WAL_FILE_NAME = "tagInvertedIndex.log"; + private static final int INSERT = 1; + private static final int DELETE = 2; + private final String schemaDirPath; + private File walFile; + private WALWriter walWriter; + private WALReader walReader; + + public WALManager(String schemaDirPath) throws IOException { + this.schemaDirPath = schemaDirPath; + initFile(schemaDirPath); + walWriter = new WALWriter(walFile, false); + walReader = new WALReader(walFile, new WALEntry()); + } + + private void initFile(String schemaDirPath) throws IOException { + File schemaDir = new File(schemaDirPath); + schemaDir.mkdirs(); + walFile = new File(this.schemaDirPath, WAL_FILE_NAME); + if (!walFile.exists()) { + walFile.createNewFile(); + } + } + + public synchronized void write(Context context) throws IOException { + switch (context.getType()) { + case INSERT: + process((InsertContext) context); + break; + case DELETE: + process((DeleteContext) context); + break; + default: + break; + } + } + + public synchronized Context read() { + if (walReader.hasNext()) { + WALEntry walEntry = (WALEntry) walReader.next(); + if (walEntry.getType() == INSERT) { + return generateInsertContext(walEntry); + } + if (walEntry.getType() == DELETE) { + return generateDeleteContext(walEntry); + } + } + return new Context(); + } + + private InsertContext generateInsertContext(WALEntry walEntry) { + InsertContext insertContext = new InsertContext(walEntry.getDeviceID(), walEntry.getKeys()); + return insertContext; + } + + private DeleteContext generateDeleteContext(WALEntry walEntry) { + DeleteContext deleteContext = new DeleteContext(walEntry.getDeviceID(), walEntry.getKeys()); + return deleteContext; + } + + private void process(InsertContext insertContext) throws IOException { + List objects = insertContext.getKeys(); + List keys = new ArrayList<>(); + for (Object o : objects) { + keys.add((String) o); + } + WALEntry walEntry = new WALEntry(INSERT, keys, (Integer) insertContext.getValue()); + walWriter.write(walEntry); + } + + private void process(DeleteContext deleteContext) throws IOException { + List objects = deleteContext.getKeys(); + List keys = new ArrayList<>(); + for (Object o : objects) { + keys.add((String) o); + } + WALEntry walEntry = new WALEntry(DELETE, keys, (Integer) deleteContext.getValue()); + walWriter.write(walEntry); + } + + public void close() throws IOException { + walWriter.close(); + walReader.close(); + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index a927c2ebbce79..eba4c1067d3f3 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; +import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.tsfile.utils.Pair; @@ -25,6 +26,7 @@ import org.junit.Before; import org.junit.Test; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -36,42 +38,55 @@ public class TagInvertedIndexTest { private String[][] record = new String[][] { - {"tag1=q", "tag2=a", "1"}, - {"tag1=q", "tag2=s", "2"}, - {"tag1=q", "tag2=a", "tag3=z", "3"}, - {"tag1=q", "tag3=v", "4"}, - {"tag1=q", "tag2=s", "5"}, - {"tag1=w", "tag2=d", "6"}, - {"tag1=q", "tag2=d", "tag3=e", "7"}, - {"tag1=t", "tag2=g", "8"}, - {"tag1=r", "tag2=d", "9"}, - {"tag1=t", "tag2=f", "10"}, - {"tag1=t", "tag2=h", "11"}, - {"tag1=q", "tag2=a", "tag3=l", "12"}, - {"tag1=y", "tag2=j", "13"}, - {"tag1=u", "tag2=k", "14"}, - {"tag1=q", "tag2=a", "tag3=x", "15"}, - {"tag1=q", "tag2=a", "tag4=z", "16"}, - {"tag1=y", "tag2=a", "tag4=z", "17"}, - {"tag1=q", "tag2=b", "tag3=x", "18"}, + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, }; private int numOfDeviceIdsInMemTable; private TagInvertedIndex tagInvertedIndex; + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testTagIndex"; + + private String schemaDir; + @Before public void setUp() throws Exception { numOfDeviceIdsInMemTable = IoTDBDescriptor.getInstance().getConfig().getNumOfDeviceIdsInMemTable(); IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(3); - tagInvertedIndex = new TagInvertedIndex(); + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); } @After public void tearDown() throws Exception { IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + tagInvertedIndex.clear(); tagInvertedIndex = null; + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); } public void addTags() { From d23d8ca86f9ab8c5cb5a4397ef373dc910b0c000 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:13:35 +0800 Subject: [PATCH 14/53] impl RecoverManager --- .../org/apache/iotdb/lsm/context/Context.java | 12 +++++ .../iotdb/lsm/context/DeleteContext.java | 13 ++++- .../iotdb/lsm/context/InsertContext.java | 18 +++++-- .../iotdb/lsm/manager/BasicLsmManager.java | 6 +-- .../tagIndex/ITagInvertedIndex.java | 8 +++ .../tagIndex/TagInvertedIndex.java | 52 +++++++++++++++---- .../tagIndex/deletion/DeletionManager.java | 9 +++- .../tagIndex/insertion/InsertionManager.java | 7 +-- .../tagIndex/recover/RecoverManager.java | 19 ++++++- .../tagIndex/wal/WALManager.java | 13 ++++- .../tagIndex/TagInvertedIndexTest.java | 30 ++++++++--- 11 files changed, 153 insertions(+), 34 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java index 305b31747f6c3..3f369b3017734 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -41,12 +41,16 @@ public class Context { // 返回值 Object result; + // 是否正在recover + boolean recover; + public Context() { accessStrategy = new PreOrderAccessStrategy(); type = ContextType.NONE; level = 0; threadNums = 1; levelUpperBound = Integer.MAX_VALUE; + recover = false; } public void setLevel(int level) { @@ -96,4 +100,12 @@ public int getLevelUpperBound() { public void setLevelUpperBound(int levelUpperBound) { this.levelUpperBound = levelUpperBound; } + + public boolean isRecover() { + return recover; + } + + public void setRecover(boolean recover) { + this.recover = recover; + } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java index 2b99196b0c3aa..9b4b30ac41a75 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java @@ -30,6 +30,12 @@ public class DeleteContext extends Context { Object value; + public DeleteContext() { + super(); + type = ContextType.DELETE; + accessStrategy = new PostOrderAccessStrategy(); + } + public DeleteContext(Object value, Object... ks) { super(); this.value = value; @@ -39,11 +45,14 @@ public DeleteContext(Object value, Object... ks) { accessStrategy = new PostOrderAccessStrategy(); } - public DeleteContext(Object value,List keys){ - this.value = value; + public void setKeys(List keys) { this.keys = keys; } + public void setValue(Object value) { + this.value = value; + } + public Object getKey() { return keys.get(level); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java index e8d00da4979d7..b09d1540cbc47 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java @@ -30,23 +30,31 @@ public class InsertContext extends Context { Object value; + public InsertContext() { + super(); + type = ContextType.INSERT; + accessStrategy = new PreOrderAccessStrategy(); + } + public InsertContext(Object value, Object... keys) { super(); this.value = value; this.keys = new ArrayList<>(); this.keys.addAll(Arrays.asList(keys)); - level = 0; type = ContextType.INSERT; accessStrategy = new PreOrderAccessStrategy(); } - public InsertContext(Object value,List keys){ - this.value = value; + public Object getKey() { + return keys.get(level); + } + + public void setKeys(List keys) { this.keys = keys; } - public Object getKey() { - return keys.get(level); + public void setValue(Object value) { + this.value = value; } public List getKeys() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java index a5b91439662aa..1855f04c0fe48 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -21,18 +21,16 @@ import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -import java.io.IOException; - public class BasicLsmManager implements LsmManager { LevelProcess levelProcess; public void preProcess(T root, C context) throws Exception {} - public void postProcess(T root, C context) throws Exception{} + public void postProcess(T root, C context) throws Exception {} @Override - public void process(T root, C context) throws Exception{ + public void process(T root, C context) throws Exception { preProcess(root, context); levelProcess.process(root, context); postProcess(root, context); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index cf5e1a19c0ea7..d5269bbce987e 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -18,12 +18,20 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; + import java.util.List; import java.util.Map; public interface ITagInvertedIndex { + + void addTags(InsertContext context); + void addTags(Map tags, int id); + void removeTags(DeleteContext context); + void removeTags(Map tags, int id); List getMatchedIDs(Map tags); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 71c2ee923106a..b76ed104d85fa 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -31,7 +31,6 @@ import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.context.QueryContext; -import org.apache.iotdb.lsm.wal.WALWriter; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,25 +79,57 @@ public TagInvertedIndex(String schemaDirPath) throws IOException { recover(); } - public synchronized void recover(){ + public synchronized void recover() { recoverManager.recover(this); } + @Override + public synchronized void addTags(InsertContext context) { + int id = (int) context.getValue(); + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); + } + MemTable memTable = workingMemTable; + maxDeviceID = id; + try { + insertionManager.process(memTable, context); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + @Override public synchronized void addTags(Map tags, int id) { - MemTable memTable = null; if (!inWorkingMemTable(id)) { workingMemTable.setStatus(MemTable.IMMUTABLE); immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); workingMemTable = new MemTable(MemTable.WORKING); } - memTable = workingMemTable; + MemTable memTable = workingMemTable; maxDeviceID = id; try { for (Map.Entry tag : tags.entrySet()) { addTag(memTable, tag.getKey(), tag.getValue(), id); } - }catch (Exception e){ + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + @Override + public void removeTags(DeleteContext context) { + int id = (int) context.getValue(); + MemTable memTable = null; + if (inWorkingMemTable(id)) { + memTable = workingMemTable; + } else { + memTable = immutableMemTables.get(id / numOfDeviceIdsInMemTable); + } + try { + deletionManager.process(memTable, context); + } catch (Exception e) { logger.error(e.getMessage()); } } @@ -106,7 +137,6 @@ public synchronized void addTags(Map tags, int id) { @Override public synchronized void removeTags(Map tags, int id) { List memTables = new ArrayList<>(); - // 出现乱序 if (inWorkingMemTable(id)) { memTables.add(workingMemTable); } else { @@ -116,7 +146,7 @@ public synchronized void removeTags(Map tags, int id) { for (Map.Entry tag : tags.entrySet()) { removeTag(memTables, tag.getKey(), tag.getValue(), id); } - }catch (Exception e){ + } catch (Exception e) { logger.error(e.getMessage()); } } @@ -135,7 +165,7 @@ public synchronized List getMatchedIDs(Map tags) { else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); i++; } - }catch (Exception e){ + } catch (Exception e) { logger.error(e.getMessage()); } return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); @@ -164,14 +194,16 @@ private void addTag(MemTable memTable, String tagKey, String tagValue, int id) t insertionManager.process(memTable, insertContext); } - private void removeTag(List memTables, String tagKey, String tagValue, int id) throws Exception { + private void removeTag(List memTables, String tagKey, String tagValue, int id) + throws Exception { DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); for (MemTable memTable : memTables) { deletionManager.process(memTable, deleteContext); } } - private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) throws Exception { + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) + throws Exception { QueryContext queryContext = new QueryContext(tagKey, tagValue); for (MemTable memTable : memTables) { queryManager.process(memTable, queryContext); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index 0d8e62a769f67..f31b6461b2ca3 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -23,8 +23,6 @@ import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; -import java.io.IOException; - public class DeletionManager extends BasicLsmManager { WALManager walManager; @@ -34,6 +32,13 @@ public DeletionManager(WALManager walManager) { initLevelProcess(); } + @Override + public void preProcess(MemTable root, DeleteContext context) throws Exception { + if (!context.isRecover()) { + walManager.write(context); + } + } + private void initLevelProcess() { this.nextLevel(new MemTableDeletion()) .nextLevel(new MemChunkGroupDeletion()) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java index 4b4dd29ee8c62..0e203a3d646ba 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.InsertContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; @@ -30,14 +29,16 @@ public class InsertionManager extends BasicLsmManager { private WALManager walManager; - public InsertionManager(WALManager walManager){ + public InsertionManager(WALManager walManager) { this.walManager = walManager; initLevelProcess(); } @Override public void preProcess(MemTable root, InsertContext context) throws IOException { - walManager.write(context); + if (!context.isRecover()) { + walManager.write(context); + } } private void initLevelProcess() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 2c3d38bc06278..98b4ee5745059 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -20,6 +20,9 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; +import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.InsertContext; public class RecoverManager { @@ -29,5 +32,19 @@ public RecoverManager(WALManager walManager) { this.walManager = walManager; } - public void recover(TagInvertedIndex tagInvertedIndex) {} + public void recover(TagInvertedIndex tagInvertedIndex) { + while (true) { + Context context = walManager.read(); + switch (context.getType()) { + case INSERT: + tagInvertedIndex.addTags((InsertContext) context); + break; + case DELETE: + tagInvertedIndex.removeTags((DeleteContext) context); + break; + default: + return; + } + } + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java index 88bdf68dd3143..b0ac296f6c15f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -67,6 +67,7 @@ public synchronized void write(Context context) throws IOException { } } + // 用于recover public synchronized Context read() { if (walReader.hasNext()) { WALEntry walEntry = (WALEntry) walReader.next(); @@ -81,12 +82,22 @@ public synchronized Context read() { } private InsertContext generateInsertContext(WALEntry walEntry) { - InsertContext insertContext = new InsertContext(walEntry.getDeviceID(), walEntry.getKeys()); + InsertContext insertContext = new InsertContext(); + List objects = new ArrayList<>(); + objects.addAll(walEntry.getKeys()); + insertContext.setKeys(objects); + insertContext.setValue(walEntry.getDeviceID()); + insertContext.setRecover(true); return insertContext; } private DeleteContext generateDeleteContext(WALEntry walEntry) { DeleteContext deleteContext = new DeleteContext(walEntry.getDeviceID(), walEntry.getKeys()); + List objects = new ArrayList<>(); + objects.addAll(walEntry.getKeys()); + deleteContext.setKeys(objects); + deleteContext.setValue(walEntry.getDeviceID()); + deleteContext.setRecover(true); return deleteContext; } diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index eba4c1067d3f3..f193862a68592 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -110,8 +111,6 @@ public void removeTags() { @Test public void getMatchedIDs() { addTags(); - System.out.println("------------addTags------------"); - System.out.println(tagInvertedIndex); Map tags1 = new HashMap<>(); tags1.put("tag1", "q"); @@ -128,8 +127,7 @@ public void getMatchedIDs() { assertEquals(verify, ids); removeTags(); - System.out.println("------------removeTags------------"); - System.out.println(tagInvertedIndex); + ids = tagInvertedIndex.getMatchedIDs(tags1); verify = Arrays.asList(3, 5, 7, 15, 16, 18); assertEquals(verify, ids); @@ -137,9 +135,29 @@ public void getMatchedIDs() { ids = tagInvertedIndex.getMatchedIDs(tags2); verify = Arrays.asList(3, 15, 16); assertEquals(verify, ids); + } + + @Test + public void testRecover() throws IOException { + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); - System.out.println("------------query------------"); - System.out.println(tagInvertedIndex); + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + addTags(); + removeTags(); + + tagInvertedIndex.clear(); + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); } private List, Integer>> generateTags() { From b770521295b6a8782795031cac0087fdafe9538a Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:14:37 +0800 Subject: [PATCH 15/53] impl RecoverManager --- .../db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index b76ed104d85fa..9ae8b0ddb2dce 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -185,7 +185,7 @@ public String toString() { + '}'; } - private synchronized boolean inWorkingMemTable(int id) { + private boolean inWorkingMemTable(int id) { return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; } From 9e25cc5c6dbabf61e08145222a1f749e9b235c31 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:15:47 +0800 Subject: [PATCH 16/53] impl RecoverManager --- .../tagSchemaRegion/tagIndex/recover/RecoverManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 98b4ee5745059..906fafb25503b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover; -import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.ITagInvertedIndex; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.context.DeleteContext; @@ -32,7 +32,7 @@ public RecoverManager(WALManager walManager) { this.walManager = walManager; } - public void recover(TagInvertedIndex tagInvertedIndex) { + public void recover(ITagInvertedIndex tagInvertedIndex) { while (true) { Context context = walManager.read(); switch (context.getType()) { From 76759536d14dbceb9720e26ef8c8c1cd6a2e52b2 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:20:05 +0800 Subject: [PATCH 17/53] impl RecoverManager --- .../tagSchemaRegion/tagIndex/deletion/DeletionManager.java | 2 +- .../tagSchemaRegion/tagIndex/recover/RecoverManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index f31b6461b2ca3..211eed0bbe944 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -25,7 +25,7 @@ public class DeletionManager extends BasicLsmManager { - WALManager walManager; + private WALManager walManager; public DeletionManager(WALManager walManager) { this.walManager = walManager; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 906fafb25503b..ec9fd74729565 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -26,7 +26,7 @@ public class RecoverManager { - WALManager walManager; + private WALManager walManager; public RecoverManager(WALManager walManager) { this.walManager = walManager; From f96336db384951f8bb333e3204ad8f2fbd9ee31f Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 14 Sep 2022 20:36:38 +0800 Subject: [PATCH 18/53] impl RecoverManager --- lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java | 1 + .../db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java index a44b8f9d243c5..4410b6f8a2007 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; public abstract class WALRecord implements Cloneable { + public abstract void serialize(ByteBuffer buffer); public abstract void deserialize(DataInputStream stream) throws IOException; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java index e70ad62100b10..6cf24548492a2 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -34,9 +34,10 @@ public class WALEntry extends WALRecord { private int deviceID; - public WALEntry() {} + public WALEntry() {super();} public WALEntry(int type, List keys, int deviceID) { + super(); this.type = type; this.keys = keys; this.deviceID = deviceID; From 53c3f4a337489b85fac48af4c3f3251c2b9bb3e4 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Mon, 19 Sep 2022 23:28:42 +0800 Subject: [PATCH 19/53] spotless apply --- schema-engine-tag/pom.xml | 1 + .../tagSchemaRegion/MockTagSchemaRegion.java | 325 ------------------ .../tagIndex/wal/WALEntry.java | 4 +- 3 files changed, 4 insertions(+), 326 deletions(-) delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml index c5b8f565525a0..731ead4ad0c6a 100644 --- a/schema-engine-tag/pom.xml +++ b/schema-engine-tag/pom.xml @@ -39,6 +39,7 @@ org.apache.iotdb iotdb-server ${project.version} + provided diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java deleted file mode 100644 index 184ce789b99cd..0000000000000 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.metadata.tagSchemaRegion; - -import org.apache.iotdb.common.rpc.thrift.TSchemaNode; -import org.apache.iotdb.commons.consensus.SchemaRegionId; -import org.apache.iotdb.commons.exception.MetadataException; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.exception.metadata.PathNotExistException; -import org.apache.iotdb.db.metadata.LocalSchemaProcessor; -import org.apache.iotdb.db.metadata.mnode.IMNode; -import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; -import org.apache.iotdb.db.metadata.path.MeasurementPath; -import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; -import org.apache.iotdb.db.metadata.template.Template; -import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; -import org.apache.iotdb.db.qp.physical.crud.InsertPlan; -import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; -import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; -import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; -import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; -import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; -import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; -import org.apache.iotdb.db.query.context.QueryContext; -import org.apache.iotdb.db.query.dataset.ShowDevicesResult; -import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.utils.Pair; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; - -public class MockTagSchemaRegion implements ISchemaRegion { - - @Override - public void init() throws MetadataException {} - - @Override - public void clear() {} - - @Override - public void forceMlog() {} - - @Override - public SchemaRegionId getSchemaRegionId() { - return null; - } - - @Override - public String getStorageGroupFullPath() { - return null; - } - - @Override - public void deleteSchemaRegion() throws MetadataException {} - - @Override - public boolean createSnapshot(File snapshotDir) { - return false; - } - - @Override - public void loadSnapshot(File latestSnapshotRootDir) {} - - @Override - public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {} - - @Override - public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {} - - @Override - public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return null; - } - - @Override - public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {} - - @Override - public boolean isPathExist(PartialPath path) throws MetadataException { - return false; - } - - @Override - public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public int getAllTimeseriesCount( - PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public int getAllTimeseriesCount( - PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) - throws MetadataException { - return 0; - } - - @Override - public Map getMeasurementCountGroupByLevel( - PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { - return null; - } - - @Override - public Map getMeasurementCountGroupByLevel( - PartialPath pathPattern, - int level, - boolean isPrefixMatch, - String key, - String value, - boolean isContains) - throws MetadataException { - return null; - } - - @Override - public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public List getNodesListInGivenLevel( - PartialPath pathPattern, - int nodeLevel, - boolean isPrefixMatch, - LocalSchemaProcessor.StorageGroupFilter filter) - throws MetadataException { - return null; - } - - @Override - public Set getChildNodePathInNextLevel(PartialPath pathPattern) - throws MetadataException { - return null; - } - - @Override - public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { - return null; - } - - @Override - public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { - return null; - } - - @Override - public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return null; - } - - @Override - public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) - throws MetadataException { - return null; - } - - @Override - public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - return null; - } - - @Override - public Pair, Integer> getMeasurementPathsWithAlias( - PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) - throws MetadataException { - return null; - } - - @Override - public List fetchSchema( - PartialPath pathPattern, Map templateMap) throws MetadataException { - return null; - } - - @Override - public Pair, Integer> showTimeseries( - ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { - return null; - } - - @Override - public List getAllMeasurementByDevicePath(PartialPath devicePath) - throws PathNotExistException { - return null; - } - - @Override - public IMNode getDeviceNode(PartialPath path) throws MetadataException { - return null; - } - - @Override - public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { - return null; - } - - @Override - public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {} - - @Override - public void upsertTagsAndAttributes( - String alias, - Map tagsMap, - Map attributesMap, - PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void addAttributes(Map attributesMap, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void addTags(Map tagsMap, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) - throws MetadataException, IOException {} - - @Override - public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) - throws MetadataException, IOException { - return null; - } - - @Override - public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( - PartialPath devicePath, - String[] measurements, - Function getDataType, - boolean aligned) - throws MetadataException { - return null; - } - - @Override - public Set getPathsSetTemplate(String templateName) throws MetadataException { - return null; - } - - @Override - public Set getPathsUsingTemplate(String templateName) throws MetadataException { - return null; - } - - @Override - public boolean isTemplateAppendable(Template template, List measurements) - throws MetadataException { - return false; - } - - @Override - public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {} - - @Override - public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {} - - @Override - public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {} - - @Override - public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) - throws MetadataException {} - - @Override - public List getPathsUsingTemplate(int templateId) throws MetadataException { - return null; - } - - @Override - public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { - return null; - } - - @Override - public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {} -} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java index 6cf24548492a2..6c14934ce1062 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -34,7 +34,9 @@ public class WALEntry extends WALRecord { private int deviceID; - public WALEntry() {super();} + public WALEntry() { + super(); + } public WALEntry(int type, List keys, int deviceID) { super(); From dfb56b011eeb39ad60124ea5dee706180d05a353 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Wed, 21 Sep 2022 09:49:25 +0800 Subject: [PATCH 20/53] add TagSchemaDescriptor --- .../org/apache/iotdb/lsm/wal/WALWriter.java | 9 +- schema-engine-tag/pom.xml | 32 ++ .../resources/conf/schema-tag.properties | 33 ++ .../src/assembly/schema-engine-tag.xml | 45 +++ .../tagSchemaRegion/TagSchemaConfig.java | 42 +++ .../tagSchemaRegion/TagSchemaDescriptor.java | 85 +++++ .../tagSchemaRegion/TagSchemaRegion.java | 325 +++++++++++++++++- .../tagIndex/TagInvertedIndex.java | 7 +- .../tagIndex/wal/WALManager.java | 16 +- .../tagIndex/TagInvertedIndexTest.java | 9 +- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 10 - .../schemaregion/TagSchemaRegionLoader.java | 112 ++++++ 12 files changed, 705 insertions(+), 20 deletions(-) create mode 100644 schema-engine-tag/src/assembly/resources/conf/schema-tag.properties create mode 100644 schema-engine-tag/src/assembly/schema-engine-tag.xml create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java index dccd8a92233db..abaf89a354dec 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java @@ -34,15 +34,18 @@ public class WALWriter implements IWALWriter { private File logFile; private FileOutputStream fileOutputStream; private FileChannel channel; - private final ByteBuffer lengthBuffer = ByteBuffer.allocate(4); - private final ByteBuffer walBuffer = ByteBuffer.allocate(10000); + private final ByteBuffer lengthBuffer; + private final ByteBuffer walBuffer; private final boolean forceEachWrite; - public WALWriter(File logFile, boolean forceEachWrite) throws FileNotFoundException { + public WALWriter(File logFile, int walBufferSize, boolean forceEachWrite) + throws FileNotFoundException { this.logFile = logFile; this.forceEachWrite = forceEachWrite; fileOutputStream = new FileOutputStream(logFile, true); channel = fileOutputStream.getChannel(); + lengthBuffer = ByteBuffer.allocate(4); + walBuffer = ByteBuffer.allocate(walBufferSize); } @Override diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml index 731ead4ad0c6a..14e61f754aa21 100644 --- a/schema-engine-tag/pom.xml +++ b/schema-engine-tag/pom.xml @@ -42,4 +42,36 @@ provided + + schema-engine-tag + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven.assembly.version} + + + + schema-engine-tag-assembly + package + + single + + + + src/assembly/schema-engine-tag.xml + + false + + + true + true + + + + + + + + diff --git a/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties b/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties new file mode 100644 index 0000000000000..bfb7df48b8ab4 --- /dev/null +++ b/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +#################### +### tag schema region Configuration +#################### + +# This configuration takes effect only when the schema engine mode is Tag. +# The mode is configured in the 'iotdb-datanode.properties'(schema_engine_mode=Tag). + +# Datatype: int +# The size of wal buffer used to store a wal record.(unit: byte) +# wal_buffer_size = 1024*1024 + +# Datatype: int +# How many device ids a memtable can insert, beyond which the memtable will become immutable +# num_of_deviceIds_in_memTable = 65536 \ No newline at end of file diff --git a/schema-engine-tag/src/assembly/schema-engine-tag.xml b/schema-engine-tag/src/assembly/schema-engine-tag.xml new file mode 100644 index 0000000000000..49647f46a7a14 --- /dev/null +++ b/schema-engine-tag/src/assembly/schema-engine-tag.xml @@ -0,0 +1,45 @@ + + + + schema-engine-tag + + dir + zip + + false + + + /lib/tag-schema-region + + org.apache.iotdb:iotdb-lsm + org.roaringbitmap:RoaringBitmap + org.apache.iotdb:schema-engine-tag + + + + + + src/assembly/resources + ${file.separator} + + + diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java new file mode 100644 index 0000000000000..ad13817596a4f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +public class TagSchemaConfig { + + private int numOfDeviceIdsInMemTable = 65536; + + private int walBufferSize = 1024 * 1024; + + public int getNumOfDeviceIdsInMemTable() { + return numOfDeviceIdsInMemTable; + } + + public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { + this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; + } + + public int getWalBufferSize() { + return walBufferSize; + } + + public void setWalBufferSize(int walBufferSize) { + this.walBufferSize = walBufferSize; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java new file mode 100644 index 0000000000000..97f7d4752b99c --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.commons.conf.IoTDBConstant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class TagSchemaDescriptor { + private static final Logger logger = LoggerFactory.getLogger(TagSchemaDescriptor.class); + + private static final String TAG_SCHEMA_CONFIG_FILE_NAME = "schema-tag.properties"; + + private final TagSchemaConfig conf = new TagSchemaConfig(); + + private TagSchemaDescriptor() { + loadProperties(); + } + + public static TagSchemaDescriptor getInstance() { + return TagSchemaDescriptorHolder.INSTANCE; + } + + private void loadProperties() { + String iotdbHomePath = System.getProperty(IoTDBConstant.IOTDB_HOME, null); + String tagSchemaConfigPath = + iotdbHomePath + + File.separatorChar + + "conf" + + File.separatorChar + + TAG_SCHEMA_CONFIG_FILE_NAME; + try (InputStream in = new BufferedInputStream(new FileInputStream(tagSchemaConfigPath))) { + Properties properties = new Properties(); + properties.load(in); + conf.setWalBufferSize( + Integer.parseInt( + properties.getProperty("wal_buffer_size", String.valueOf(conf.getWalBufferSize())))); + conf.setNumOfDeviceIdsInMemTable( + Integer.parseInt( + properties.getProperty( + "num_of_deviceIds_in_memTable", + String.valueOf(conf.getNumOfDeviceIdsInMemTable())))); + } catch (FileNotFoundException e) { + logger.warn("Fail to find tag schema region config file {}", tagSchemaConfigPath); + } catch (IOException e) { + logger.warn("Cannot load tag schema region config file, use default configuration"); + } + } + + public TagSchemaConfig getTagSchemaConfig() { + return conf; + } + + private static class TagSchemaDescriptorHolder { + + private static final TagSchemaDescriptor INSTANCE = new TagSchemaDescriptor(); + + private TagSchemaDescriptorHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index 2f04d7c095a73..857575d1efca5 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -18,4 +18,327 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion; -public class TagSchemaRegion {} +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.Pair; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +public class TagSchemaRegion implements ISchemaRegion { + + + @Override + public void init() throws MetadataException {} + + @Override + public void clear() {} + + @Override + public void forceMlog() {} + + @Override + public SchemaRegionId getSchemaRegionId() { + return null; + } + + @Override + public String getStorageGroupFullPath() { + return null; + } + + @Override + public void deleteSchemaRegion() throws MetadataException {} + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) {} + + @Override + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {} + + @Override + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {} + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + return 0; + } + + @Override + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public List fetchSchemaBlackList(PathPatternTree patternTree) + throws MetadataException { + return null; + } + + @Override + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {} + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + return false; + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + return null; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + return null; + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + return null; + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + return null; + } + + @Override + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + return null; + } + + @Override + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + return null; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + @Override + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + return null; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + return null; + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + return null; + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {} + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException {} + + @Override + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + return null; + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + return null; + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + return null; + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + return false; + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {} + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {} + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {} + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + return null; + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 9ae8b0ddb2dce..ed62c3d2006d7 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -21,6 +21,8 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; @@ -48,6 +50,9 @@ public class TagInvertedIndex implements ITagInvertedIndex { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final TagSchemaConfig tagSchemaConfig = + TagSchemaDescriptor.getInstance().getTagSchemaConfig(); + private final InsertionManager insertionManager; private final DeletionManager deletionManager; @@ -74,7 +79,7 @@ public TagInvertedIndex(String schemaDirPath) throws IOException { queryManager = new QueryManager(); workingMemTable = new MemTable(MemTable.WORKING); immutableMemTables = new HashMap<>(); - numOfDeviceIdsInMemTable = config.getNumOfDeviceIdsInMemTable(); + numOfDeviceIdsInMemTable = tagSchemaConfig.getNumOfDeviceIdsInMemTable(); maxDeviceID = 0; recover(); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java index b0ac296f6c15f..65a74bba89144 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.context.InsertContext; @@ -30,18 +32,28 @@ import java.util.List; public class WALManager { - private static final String WAL_FILE_NAME = "tagInvertedIndex.log"; + private static final String WAL_FILE_NAME = "tag_inverted_index.log"; + private static final int INSERT = 1; + private static final int DELETE = 2; + + private final TagSchemaConfig tagSchemaConfig = + TagSchemaDescriptor.getInstance().getTagSchemaConfig(); + private final String schemaDirPath; + private File walFile; + private WALWriter walWriter; + private WALReader walReader; public WALManager(String schemaDirPath) throws IOException { this.schemaDirPath = schemaDirPath; initFile(schemaDirPath); - walWriter = new WALWriter(walFile, false); + int walBufferSize = tagSchemaConfig.getWalBufferSize(); + walWriter = new WALWriter(walFile, walBufferSize, false); walReader = new WALReader(walFile, new WALEntry()); } diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java index f193862a68592..e8599f4b985c1 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; import org.apache.iotdb.tsfile.utils.Pair; import org.junit.After; @@ -74,8 +75,8 @@ public class TagInvertedIndexTest { @Before public void setUp() throws Exception { numOfDeviceIdsInMemTable = - IoTDBDescriptor.getInstance().getConfig().getNumOfDeviceIdsInMemTable(); - IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(3); + TagSchemaDescriptor.getInstance().getTagSchemaConfig().getNumOfDeviceIdsInMemTable(); + TagSchemaDescriptor.getInstance().getTagSchemaConfig().setNumOfDeviceIdsInMemTable(3); schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; schemaRegionDirPath = storageGroupDirPath + File.separator + 0; @@ -84,7 +85,9 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - IoTDBDescriptor.getInstance().getConfig().setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + TagSchemaDescriptor.getInstance() + .getTagSchemaConfig() + .setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); tagInvertedIndex.clear(); tagInvertedIndex = null; FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index e591fe56aa09b..50d69dc5aa53a 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -866,8 +866,6 @@ public class IoTDBConfig { */ private boolean enableIDTableLogFile = false; - private int numOfDeviceIdsInMemTable = 65536; - /** whether to use persistent schema mode */ private String schemaEngineMode = "Memory"; @@ -2869,14 +2867,6 @@ public void setEnableIDTableLogFile(boolean enableIDTableLogFile) { this.enableIDTableLogFile = enableIDTableLogFile; } - public int getNumOfDeviceIdsInMemTable() { - return numOfDeviceIdsInMemTable; - } - - public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { - this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; - } - public String getSchemaEngineMode() { return schemaEngineMode; } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java new file mode 100644 index 0000000000000..46a8056d05b89 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.schemaregion; + +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.LinkedList; +import java.util.List; + +public class TagSchemaRegionLoader { + private static final Logger LOGGER = LoggerFactory.getLogger(TagSchemaRegionLoader.class); + private static URLClassLoader urlClassLoader = null; + private static final String TAG_SCHEMA_REGION_CLASS_NAME = + "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaRegion"; + private static final String TAG_SCHEMA_CONF_DESCRIPTOR_CLASS_NAME = + "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor"; + private static final String LIB_PATH = + ".." + File.separator + "lib" + File.separator + "tag-schema-region" + File.separator; + + public TagSchemaRegionLoader() {} + + /** + * Load the jar files for RSchemaRegion and create an instance of it. The jar files should be + * located in "../lib/rschema-region". If jar files cannot be found, the function will return + * null. + * + * @param storageGroup + * @param schemaRegionId + * @param node + * @return + */ + public ISchemaRegion loadRSchemaRegion( + PartialPath storageGroup, SchemaRegionId schemaRegionId, IStorageGroupMNode node) { + ISchemaRegion region = null; + LOGGER.info("Creating instance for schema-engine-rocksdb"); + try { + loadRSchemaRegionJar(); + Class classForTagSchemaRegion = urlClassLoader.loadClass(TAG_SCHEMA_REGION_CLASS_NAME); + Class classForTagSchemaDescriptor = + urlClassLoader.loadClass(TAG_SCHEMA_CONF_DESCRIPTOR_CLASS_NAME); + Constructor constructor = + classForTagSchemaRegion.getConstructor( + PartialPath.class, + SchemaRegionId.class, + IStorageGroupMNode.class, + classForTagSchemaDescriptor); + Object rSchemaLoader = classForTagSchemaDescriptor.getConstructor().newInstance(); + region = + (ISchemaRegion) + constructor.newInstance(storageGroup, schemaRegionId, node, rSchemaLoader); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | InstantiationException + | IllegalAccessException + | MalformedURLException + | RuntimeException e) { + LOGGER.error("Cannot initialize RSchemaRegion", e); + return null; + } + return region; + } + + /** + * Load the jar files for rocksdb and RSchemaRegion. The jar files should be located in directory + * "../lib/rschema-region". If the jar files have been loaded, it will do nothing. + */ + private void loadRSchemaRegionJar() throws MalformedURLException { + LOGGER.info("Loading jar for schema-engine-rocksdb"); + if (urlClassLoader == null) { + File[] jars = new File(LIB_PATH).listFiles(); + if (jars == null) { + throw new RuntimeException( + String.format("Cannot get jars from %s", new File(LIB_PATH).getAbsolutePath())); + } + List dependentJars = new LinkedList<>(); + for (File jar : jars) { + if (jar.getName().endsWith(".jar")) { + dependentJars.add(new URL("file:" + jar.getAbsolutePath())); + } + } + urlClassLoader = new URLClassLoader(dependentJars.toArray(new URL[] {})); + } + } +} From 1a649dc30470c71bca6b2b0f1fd7ef5ef3a83827 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 22 Sep 2022 14:33:33 +0800 Subject: [PATCH 21/53] add DeviceIDList --- .../org/apache/iotdb/lsm/wal/WALReader.java | 4 +- .../org/apache/iotdb/lsm/wal/WALRecord.java | 2 +- schema-engine-tag/README.md | 37 + .../tagSchemaRegion/MockTagSchemaRegion.java | 928 ++++++++++++++++++ .../tagSchemaRegion/TagSchemaConfig.java | 10 + .../tagSchemaRegion/TagSchemaRegion.java | 705 +++++++++++-- .../AppendOnlyDeviceIDListFileManager.java | 105 ++ .../deviceidlist/DeviceIDList.java | 70 ++ .../deviceidlist/IDeviceIDList.java | 39 + .../tagIndex/TagInvertedIndex.java | 40 +- .../tagIndex/recover/RecoverManager.java | 5 + .../utils/MeasurementPathUtils.java | 62 ++ .../utils/PathTagConverterUtils.java | 49 + .../utils/ShowTimeSeriesResultUtils.java | 54 + .../deviceidlist/DeviceIDListTest.java | 122 +++ .../resources/conf/iotdb-datanode.properties | 15 +- .../idtable/entry/DiskSchemaEntry.java | 4 + .../metadata/schemaregion/SchemaEngine.java | 6 +- .../schemaregion/TagSchemaRegionLoader.java | 37 +- 19 files changed, 2187 insertions(+), 107 deletions(-) create mode 100644 schema-engine-tag/README.md create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java create mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java index a3fef56b0860d..42932301d0dc4 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -65,10 +65,10 @@ public boolean hasNext() { nextRecord = prototype.clone(); nextRecord.deserialize(logStream); } catch (EOFException e) { - logger.info(""); + logger.info(e.getMessage()); return false; } catch (IOException e) { - logger.warn(""); + logger.warn(e.getMessage()); fileCorrupted = true; return false; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java index 4410b6f8a2007..4311b5debfd2a 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -33,7 +33,7 @@ public WALRecord clone() { try { return (WALRecord) super.clone(); } catch (CloneNotSupportedException e) { - throw new AssertionError(); + throw new AssertionError(e.getMessage()); } } } diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md new file mode 100644 index 0000000000000..70185b9c79731 --- /dev/null +++ b/schema-engine-tag/README.md @@ -0,0 +1,37 @@ + +`TagSchemaRegion` is an implementation of `SchemaRegion`. + +# How To Use + +Firstly, you should package **schema-engine-rocksdb** by the following command: + +```shell +mvn clean package -pl schema-engine-tag -am -DskipTests +``` + +After that, you can get a **conf** directory and a **lib** directory in +schema-engine-tag/target/schema-engine-tag. Copy the file in the conf directory to the conf directory of server, +and copy the files in the lib directory to the lib directory of server. + +Then, open the **iotdb-datanode.properties** in the conf directory of server, and set the `schema_engine_mode` to +Tag, set the `enable_id_table` to true. Restart the IoTDB, the system will use `TagSchemaRegion` to manage +the metadata. \ No newline at end of file diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java new file mode 100644 index 0000000000000..bec4ba83947d7 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -0,0 +1,928 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.IDTable; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; +import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.mnode.EntityMNode; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +public class MockTagSchemaRegion implements ISchemaRegion { + + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final String TAIL = ".**"; + private final IStorageGroupMNode storageGroupMNode; + private String storageGroupFullPath; + private SchemaRegionId schemaRegionId; + + private Map>> tagInvertedIndex; + + private List deviceIDS; + + private IDTable idTable; + + private final ISeriesNumerLimiter seriesNumerLimiter; + + public MockTagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode storageGroupMNode, + ISeriesNumerLimiter seriesNumerLimiter) + throws MetadataException { + + storageGroupFullPath = storageGroup.getFullPath(); + this.schemaRegionId = schemaRegionId; + this.storageGroupMNode = storageGroupMNode; + this.deviceIDS = new ArrayList<>(); + this.seriesNumerLimiter = seriesNumerLimiter; + tagInvertedIndex = new ConcurrentHashMap<>(); + idTable = IDTableManager.getInstance().getIDTable(storageGroup); + init(); + } + + @NotNull + private Map pathToTags(String path) { + if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); + String devicePath = path.substring(storageGroupFullPath.length() + 1); + String[] tags = devicePath.split("\\."); + Map tagsMap = new TreeMap<>(); + for (int i = 0; i < tags.length; i += 2) { + tagsMap.put(tags[i], tags[i + 1]); + } + return tagsMap; + } + + public String tagsToPath(Map tags) { + StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); + for (String tagKey : tags.keySet()) { + stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); + } + return stringBuilder.toString(); + } + + @Override + public void init() throws MetadataException { + if (!config.isEnableIDTableLogFile() + && config.getDeviceIDTransformationMethod().equals("SHA256")) { + throw new MetadataException( + "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); + } + } + + @Override + public void clear() { + return; + } + + @Override + public void forceMlog() { + return; + } + + @Override + public SchemaRegionId getSchemaRegionId() { + return schemaRegionId; + } + + @Override + public String getStorageGroupFullPath() { + return storageGroupFullPath; + } + + @Override + public void deleteSchemaRegion() throws MetadataException { + return; + } + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) { + return; + } + + private void createTagInvertedIndex(PartialPath devicePath) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + Map tagsMap = pathToTags(devicePath.getFullPath()); + + deviceIDS.add(deviceID); + + for (String tagkey : tagsMap.keySet()) { + String tagValue = tagsMap.get(tagkey); + Map> tagkeyMap = + tagInvertedIndex.computeIfAbsent(tagkey, key -> new HashMap<>()); + List ids = tagkeyMap.computeIfAbsent(tagValue, key -> new ArrayList<>()); + ids.add(deviceIDS.size() - 1); + } + } + + private void createTimeseries( + PartialPath path, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressor, + Map props) + throws MetadataException { + createTimeseries( + new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); + } + + private void createAlignedTimeSeries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries( + new CreateAlignedTimeSeriesPlan( + prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); + } + + @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { + PartialPath devicePath = plan.getPath().getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags) + "." + plan.getPath().getMeasurement()); + plan.setPath(path); + devicePath = plan.getPath().getDevicePath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { + throw new PathAlreadyExistException( + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } + } + idTable.createTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { + PartialPath devicePath = plan.getPrefixPath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags)); + plan.setPrefixPath(path); + devicePath = plan.getPrefixPath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (!deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", + devicePath.getFullPath()); + } else { + List measurements = plan.getMeasurements(); + List dataTypes = plan.getDataTypes(); + List encodings = plan.getEncodings(); + List compressors = plan.getCompressors(); + + List tmpMeasurements = new LinkedList<>(); + List tmpDataTypes = new LinkedList<>(); + List tmpEncodings = new LinkedList<>(); + List tmpCompressors = new LinkedList<>(); + for (int i = 0; i < measurements.size(); i++) { + String measurement = measurements.get(i); + if (!deviceEntry.getMeasurementMap().containsKey(measurement)) { + tmpMeasurements.add(measurements.get(i)); + tmpDataTypes.add(dataTypes.get(i)); + tmpEncodings.add(encodings.get(i)); + tmpCompressors.add(compressors.get(i)); + } + } + if (tmpMeasurements.size() == 0) + throw new PathAlreadyExistException(devicePath.getFullPath()); + plan.setMeasurements(tmpMeasurements); + plan.setDataTypes(tmpDataTypes); + plan.setEncodings(tmpEncodings); + plan.setCompressors(tmpCompressors); + } + } + idTable.createAlignedTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + return 0; + } + + @Override + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public List fetchSchemaBlackList(PathPatternTree patternTree) + throws MetadataException { + return null; + } + + @Override + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + int res = 0; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + for (IDeviceID deviceID : deviceIDs) { + res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); + } + return res; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS.size(); + } else { + return getDeviceIDsByInvertedIndex(pathPattern).size(); + } + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + Set res = new HashSet<>(); + String devicePath = pathPattern.getFullPath(); + if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry != null) { + res.add(pathPattern); + } + return res; + } + for (IDeviceID deviceID : deviceIDs) { + if (deviceID instanceof SHA256DeviceID) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map map = deviceEntry.getMeasurementMap(); + for (String m : map.keySet()) { + SchemaEntry schemaEntry = map.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new PartialPath( + diskSchemaEntry.seriesKey.substring( + 0, + diskSchemaEntry.seriesKey.length() + - diskSchemaEntry.measurementName.length() + - 1))); + break; + } + } else { + res.add(new PartialPath(deviceID.toStringID())); + } + } + return res; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select + // .. groupby level] + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + PartialPath devicePath = pathPattern.getDevicePath(); + // 批量查询.路径以".**"结尾,如： + // root.sg.tag1.a.** + // root.sg.tagx.c.tag2.v.** + // 点查询.路径不以".**",直接走IDTable,精确查询 + if (devicePath.getFullPath().endsWith(TAIL)) { + return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); + } else { + return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); + } + } + + private List getMeasurementPathsWithPointQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + String path = devicePath.getFullPath(); + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry == null) return res; + Map schemaMap = deviceEntry.getMeasurementMap(); + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + path, + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + + return res; + } + + private List getMeasurementPathsWithBatchQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); + for (IDeviceID deviceID : deviceIDs) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + MeasurementPath measurementPath = + new MeasurementPath( + new PartialPath(diskSchemaEntry.seriesKey), + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } else { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + deviceID.toStringID(), + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } + } + return res; + } + + // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + List res = getMeasurementPaths(pathPattern, isPrefixMatch); + Pair, Integer> result = new Pair<>(res, 0); + return result; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + // show 时间序列 + @Override // [iotdb/show timeseries] + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + List res = new ArrayList<>(); + Pair, Integer> result = new Pair<>(res, 0); + String path = plan.getPath().getFullPath(); + if (!path.endsWith(TAIL)) { + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry != null) { + Map measurementMap = deviceEntry.getMeasurementMap(); + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + path + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + return result; + } + List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); + for (IDeviceID deviceID : deviceIDs) { + getTimeSeriesResultOfDeviceFromIDTable(res, deviceID); + } + return result; + } + + private List getDeviceIdFromInvertedIndex(PartialPath devicePath) + throws MetadataException { + String path = devicePath.getFullPath(); + if (path.endsWith(TAIL)) { + path = path.substring(0, path.length() - TAIL.length()); + devicePath = new PartialPath(path); + } + if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS; + } else { + List res = new LinkedList<>(); + List ids = getDeviceIDsByInvertedIndex(devicePath); + if (ids.size() > 0) { + for (int id : ids) { + res.add(deviceIDS.get(id)); + } + } + return res; + } + } + + private void getTimeSeriesResultOfDeviceFromIDTable( + List res, IDeviceID deviceID) { + Map measurementMap = + idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new ShowTimeSeriesResult( + diskSchemaEntry.seriesKey, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } else { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + deviceID.toStringID() + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + } + + private List getDeviceIDsByInvertedIndex(PartialPath path) { + Map tags = pathToTags(path.getFullPath()); + List idsCollection = new ArrayList<>(tags.keySet().size()); + for (String tagKey : tags.keySet()) { + if (!tagInvertedIndex.containsKey(tagKey) + || !tagInvertedIndex.get(tagKey).containsKey(tags.get(tagKey))) { + return new ArrayList<>(); + } + List ids = tagInvertedIndex.get(tagKey).get(tags.get(tagKey)); + idsCollection.add(new ArrayList(ids)); + } + if (idsCollection.size() == 0) return new ArrayList<>(); + List ids = idsCollection.get(0); + for (int i = 1; i < idsCollection.size(); i++) { + List list = idsCollection.get(i); + ids.retainAll(list); + } + return ids; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + throw new UnsupportedOperationException(""); + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); + if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); + return new EntityMNode(storageGroupMNode, path.getFullPath()); + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + // insert data + @Override // [iotdb/insert ] + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + PartialPath devicePath = plan.getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + devicePath = new PartialPath(tagsToPath(tags)); + plan.setDevicePath(devicePath); + String[] measurementList = plan.getMeasurements(); + IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); + checkAlignedAndAutoCreateSeries(plan); + IMNode deviceMNode = getDeviceNode(devicePath); + IMeasurementMNode measurementMNode; + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (int i = 0; i < measurementList.length; i++) { + SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); + // measurementMNode = + // new MeasurementMNode( + // deviceMNode, + // measurementList[i], + // new MeasurementSchema( + // measurementList[i], + // schemaEntry.getTSDataType(), + // schemaEntry.getTSEncoding(), + // schemaEntry.getCompressionType()), + // null); + measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); + // check type is match + try { + SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); + } catch (DataTypeMismatchException mismatchException) { + if (!config.isEnablePartialInsert()) { + throw mismatchException; + } else { + // mark failed measurement + plan.markFailedMeasurementInsertion(i, mismatchException); + continue; + } + } + measurementMNodes[i] = measurementMNode; + } + plan.setDeviceID(deviceEntry.getDeviceID()); + plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); + return deviceMNode; + } + + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(getDataType.apply(i))); + + } else { + internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { + String[] measurementList = plan.getMeasurements(); + try { + if (plan.isAligned()) { + internalAlignedCreateTimeseries( + plan.getDevicePath(), + Arrays.asList(measurementList), + Arrays.asList(plan.getDataTypes())); + } else { + internalCreateTimeseries( + plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); + } + } catch (MetadataException e) { + if (!(e instanceof PathAlreadyExistException)) { + throw e; + } + } + } + + private void internalCreateTimeseries(PartialPath path, TSDataType dataType) + throws MetadataException { + createTimeseries( + path, + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor(), + Collections.emptyMap()); + } + + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, List measurements, List dataTypes) + throws MetadataException { + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + for (TSDataType dataType : dataTypes) { + encodings.add(getDefaultEncoding(dataType)); + compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { + throw new UnsupportedOperationException(""); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java index ad13817596a4f..756f0f6f446d4 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java @@ -39,4 +39,14 @@ public int getWalBufferSize() { public void setWalBufferSize(int walBufferSize) { this.walBufferSize = walBufferSize; } + + @Override + public String toString() { + return "TagSchemaConfig[" + + "numOfDeviceIdsInMemTable=" + + numOfDeviceIdsInMemTable + + ", walBufferSize=" + + walBufferSize + + "]"; + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index 857575d1efca5..c32466cd346f0 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -16,21 +16,49 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.metadata.tagSchemaRegion; import org.apache.iotdb.common.rpc.thrift.TSchemaNode; import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException; import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.IDTable; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; +import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.mnode.EntityMNode; import org.apache.iotdb.db.metadata.mnode.IMNode; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist.DeviceIDList; +import org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist.IDeviceIDList; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.MeasurementPathUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.PathTagConverterUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.ShowTimeSeriesResultUtils; import org.apache.iotdb.db.metadata.template.Template; import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; @@ -44,110 +72,320 @@ import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.dataset.ShowDevicesResult; import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + public class TagSchemaRegion implements ISchemaRegion { + private static final Logger logger = LoggerFactory.getLogger(TagSchemaRegion.class); + + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final String TAIL = ".**"; + private final IStorageGroupMNode storageGroupMNode; + private final String storageGroupFullPath; + private final SchemaRegionId schemaRegionId; + private final String schemaRegionDirPath; + + private final TagInvertedIndex tagInvertedIndex; + + private final IDeviceIDList deviceIDList; + + private final IDTable idTable; + private final ISeriesNumerLimiter seriesNumerLimiter; + + public TagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode storageGroupMNode, + ISeriesNumerLimiter seriesNumerLimiter) + throws MetadataException { + storageGroupFullPath = storageGroup.getFullPath(); + this.schemaRegionId = schemaRegionId; + String storageGroupDirPath = config.getSchemaDir() + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + schemaRegionId.getId(); + this.storageGroupMNode = storageGroupMNode; + this.seriesNumerLimiter = seriesNumerLimiter; + idTable = IDTableManager.getInstance().getIDTable(storageGroup); + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + deviceIDList = new DeviceIDList(schemaRegionDirPath); + init(); + } @Override - public void init() throws MetadataException {} + public void init() throws MetadataException { + if (!config.isEnableIDTableLogFile() + && config.getDeviceIDTransformationMethod().equals("SHA256")) { + throw new MetadataException( + "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); + } + File schemaRegionFolder = SystemFileFactory.INSTANCE.getFile(schemaRegionDirPath); + if (!schemaRegionFolder.exists()) { + if (schemaRegionFolder.mkdirs()) { + logger.info("create schema region folder {}", schemaRegionDirPath); + } else { + if (!schemaRegionFolder.exists()) { + logger.error("create schema region folder {} failed.", schemaRegionDirPath); + throw new SchemaDirCreationFailureException(schemaRegionDirPath); + } + } + } + + logger.info("initialized successfully: {}", this); + } @Override - public void clear() {} + public void clear() { + try { + tagInvertedIndex.clear(); + deviceIDList.clear(); + } catch (IOException e) { + logger.error("clear tag inverted index failed", e); + } + } @Override - public void forceMlog() {} + public void forceMlog() { + // no need to record mlog + } @Override public SchemaRegionId getSchemaRegionId() { - return null; + return schemaRegionId; } @Override public String getStorageGroupFullPath() { - return null; + return storageGroupFullPath; } @Override - public void deleteSchemaRegion() throws MetadataException {} + public void deleteSchemaRegion() throws MetadataException { + clear(); + SchemaRegionUtils.deleteSchemaRegionFolder(schemaRegionDirPath, logger); + } @Override public boolean createSnapshot(File snapshotDir) { - return false; + // todo implement this + throw new UnsupportedOperationException("Tag mode currently doesn't support snapshot feature."); } @Override - public void loadSnapshot(File latestSnapshotRootDir) {} + public void loadSnapshot(File latestSnapshotRootDir) { + // todo implement this + throw new UnsupportedOperationException("Tag mode currently doesn't support snapshot feature."); + } - @Override - public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {} + private void createTagInvertedIndex(PartialPath devicePath) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + Map tagsMap = + PathTagConverterUtils.pathToTags(storageGroupFullPath, devicePath.getFullPath()); + synchronized (deviceIDList) { + deviceIDList.add(deviceID); + tagInvertedIndex.addTags(tagsMap, deviceIDList.size() - 1); + } + } - @Override - public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {} + private List getDeviceIDsByInvertedIndex(PartialPath path) { + Map tags = + PathTagConverterUtils.pathToTags(storageGroupFullPath, path.getFullPath()); + return tagInvertedIndex.getMatchedIDs(tags); + } + + private void createTimeseries( + PartialPath path, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressor, + Map props) + throws MetadataException { + createTimeseries( + new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); + } + + private void createAlignedTimeSeries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries( + new CreateAlignedTimeSeriesPlan( + prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); + } + + @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { + PartialPath devicePath = plan.getPath().getDevicePath(); + PartialPath path = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, devicePath.getFullPath()) + + "." + + plan.getPath().getMeasurement()); + plan.setPath(path); + devicePath = plan.getPath().getDevicePath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { + throw new PathAlreadyExistException( + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } + } + idTable.createTimeseries(plan); + // write the device path for the first time + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { + PartialPath devicePath = plan.getPrefixPath(); + PartialPath path = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath( + storageGroupFullPath, devicePath.getFullPath())); + plan.setPrefixPath(path); + devicePath = plan.getPrefixPath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (!deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", + devicePath.getFullPath()); + } else { + filterExistingMeasurements(plan, deviceEntry.getMeasurementMap().keySet()); + if (plan.getMeasurements().size() == 0) + throw new PathAlreadyExistException(devicePath.getFullPath()); + } + } + idTable.createAlignedTimeseries(plan); + // write the device path for the first time + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + private void filterExistingMeasurements( + CreateAlignedTimeSeriesPlan plan, Set measurementSet) { + List measurements = plan.getMeasurements(); + List dataTypes = plan.getDataTypes(); + List encodings = plan.getEncodings(); + List compressors = plan.getCompressors(); + + List tmpMeasurements = new LinkedList<>(); + List tmpDataTypes = new LinkedList<>(); + List tmpEncodings = new LinkedList<>(); + List tmpCompressors = new LinkedList<>(); + for (int i = 0; i < measurements.size(); i++) { + String measurement = measurements.get(i); + if (!measurementSet.contains(measurement)) { + tmpMeasurements.add(measurements.get(i)); + tmpDataTypes.add(dataTypes.get(i)); + tmpEncodings.add(encodings.get(i)); + tmpCompressors.add(compressors.get(i)); + } + } + plan.setMeasurements(tmpMeasurements); + plan.setDataTypes(tmpDataTypes); + plan.setEncodings(tmpEncodings); + plan.setCompressors(tmpCompressors); + } @Override public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return null; + throw new UnsupportedOperationException("deleteTimeseries"); } @Override public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { - return 0; + throw new UnsupportedOperationException("constructSchemaBlackList"); } @Override - public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("rollbackSchemaBlackList"); + } @Override public List fetchSchemaBlackList(PathPatternTree patternTree) throws MetadataException { - return null; + throw new UnsupportedOperationException("fetchSchemaBlackList"); } @Override - public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("deleteTimeseriesInBlackList"); + } @Override - public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {} + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { + throw new UnsupportedOperationException("autoCreateDeviceMNode"); + } @Override public boolean isPathExist(PartialPath path) throws MetadataException { - return false; + throw new UnsupportedOperationException("isPathExist"); } @Override public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return 0; + int res = 0; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + for (IDeviceID deviceID : deviceIDs) { + res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); + } + return res; } @Override public int getAllTimeseriesCount( PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) throws MetadataException { - return 0; + throw new UnsupportedOperationException("getAllTimeseriesCount"); } @Override public int getAllTimeseriesCount( PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) throws MetadataException { - return 0; + throw new UnsupportedOperationException("getAllTimeseriesCount"); } @Override public Map getMeasurementCountGroupByLevel( PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMeasurementCountGroupByLevel"); } @Override @@ -159,19 +397,25 @@ public Map getMeasurementCountGroupByLevel( String value, boolean isContains) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMeasurementCountGroupByLevel"); } @Override public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return 0; + synchronized (deviceIDList) { + if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDList.size(); + } else { + return getDeviceIDsByInvertedIndex(pathPattern).size(); + } + } } @Override public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { - return 0; + throw new UnsupportedOperationException("getNodesCountInGivenLevel"); } @Override @@ -181,80 +425,270 @@ public List getNodesListInGivenLevel( boolean isPrefixMatch, LocalSchemaProcessor.StorageGroupFilter filter) throws MetadataException { - return null; + throw new UnsupportedOperationException("getNodesListInGivenLevel"); } @Override public Set getChildNodePathInNextLevel(PartialPath pathPattern) throws MetadataException { - return null; + throw new UnsupportedOperationException("getChildNodePathInNextLevel"); } @Override public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { - return null; + throw new UnsupportedOperationException("getChildNodeNameInNextLevel"); } @Override public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { - return null; + throw new UnsupportedOperationException("getBelongedDevices"); } - @Override + @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return null; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + Set matchedDevices = new HashSet<>(); + String devicePath = pathPattern.getFullPath(); + // exact query + if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry != null) { + matchedDevices.add(pathPattern); + } + return matchedDevices; + } + List devicePaths = getDevicePaths(deviceIDs); + for (String path : devicePaths) { + matchedDevices.add(new PartialPath(path)); + } + return matchedDevices; + } + + private List getDevicePaths(List deviceIDS) { + List devicePaths = new ArrayList<>(); + if (config.getDeviceIDTransformationMethod().equals("SHA256")) { + List schemaEntries = new ArrayList<>(); + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map map = deviceEntry.getMeasurementMap(); + // For each device, only one SchemaEntry needs to be obtained + for (Map.Entry entry : map.entrySet()) { + schemaEntries.add(entry.getValue()); + break; + } + } + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) { + devicePaths.add(diskSchemaEntry.getDevicePath()); + } + } else { + for (IDeviceID deviceID : deviceIDS) { + devicePaths.add(deviceID.toStringID()); + } + } + return devicePaths; + } + + private List getSchemaEntries(List deviceIDS) { + List schemaEntries = new ArrayList<>(); + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + schemaEntries.add(entry.getValue()); + } + } + return schemaEntries; + } + + private List getMeasurementPaths(List deviceIDS) + throws IllegalPathException { + List measurementPaths = new ArrayList<>(); + if (config.getDeviceIDTransformationMethod().equals("SHA256")) { + List schemaEntries = getSchemaEntries(deviceIDS); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath(diskSchemaEntry); + measurementPaths.add(measurementPath); + } + } else { + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath( + deviceID.toStringID(), entry.getKey(), entry.getValue(), deviceEntry.isAligned()); + measurementPaths.add(measurementPath); + } + } + } + return measurementPaths; } @Override public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMatchedDevices"); } - @Override + @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select + // .. groupby level] public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { - return null; - } - + PartialPath devicePath = pathPattern.getDevicePath(); + // 批量查询.路径以".**"结尾,如： + // root.sg.tag1.a.** + // root.sg.tagx.c.tag2.v.** + // 点查询.路径不以".**",直接走IDTable,精确查询 + if (devicePath.getFullPath().endsWith(TAIL)) { + return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); + } else { + return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); + } + } + + private List getMeasurementPathsWithPointQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List measurementPaths = new LinkedList<>(); + String path = + PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, devicePath.getFullPath()); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry == null) return measurementPaths; + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath( + path, entry.getKey(), entry.getValue(), deviceEntry.isAligned()); + measurementPaths.add(measurementPath); + } + return measurementPaths; + } + + private List getMeasurementPathsWithBatchQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); + return getMeasurementPaths(deviceIDs); + } + + // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] @Override public Pair, Integer> getMeasurementPathsWithAlias( PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) throws MetadataException { - return null; + List res = getMeasurementPaths(pathPattern, isPrefixMatch); + Pair, Integer> result = new Pair<>(res, 0); + return result; } @Override public List fetchSchema( PartialPath pathPattern, Map templateMap) throws MetadataException { - return null; + throw new UnsupportedOperationException("fetchSchema"); } - @Override + // show 时间序列 + @Override // [iotdb/show timeseries] public Pair, Integer> showTimeseries( ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { - return null; + List ShowTimeSeriesResults = new ArrayList<>(); + Pair, Integer> result = new Pair<>(ShowTimeSeriesResults, 0); + String path = plan.getPath().getFullPath(); + // point query + if (!path.endsWith(TAIL)) { + path = PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, path); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry != null) { + Map measurementMap = deviceEntry.getMeasurementMap(); + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, path, m, schemaEntry)); + } + } + return result; + } + // batch query + List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); + for (IDeviceID deviceID : deviceIDs) { + getTimeSeriesResultOfDeviceFromIDTable(ShowTimeSeriesResults, deviceID); + } + return result; + } + + private void getTimeSeriesResultOfDeviceFromIDTable( + List ShowTimeSeriesResults, IDeviceID deviceID) { + Map measurementMap = + idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, diskSchemaEntry.seriesKey, schemaEntry)); + } + } else { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, deviceID.toStringID(), m, schemaEntry)); + } + } + } + + private List getDeviceIdFromInvertedIndex(PartialPath devicePath) + throws MetadataException { + String path = devicePath.getFullPath(); + if (path.endsWith(TAIL)) { + path = path.substring(0, path.length() - TAIL.length()); + devicePath = new PartialPath(path); + } + synchronized (deviceIDList) { + if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDList.getAllDeviceIDS(); + } else { + List IDS = new LinkedList<>(); + List ids = getDeviceIDsByInvertedIndex(devicePath); + if (ids.size() > 0) { + for (int id : ids) { + IDS.add(deviceIDList.get(id)); + } + } + return IDS; + } + } } @Override public List getAllMeasurementByDevicePath(PartialPath devicePath) throws PathNotExistException { - return null; + throw new UnsupportedOperationException("getAllMeasurementByDevicePath"); } @Override public IMNode getDeviceNode(PartialPath path) throws MetadataException { - return null; + DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); + if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); + return new EntityMNode(storageGroupMNode, path.getFullPath()); } @Override public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMeasurementMNode"); } @Override - public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {} + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { + throw new UnsupportedOperationException("changeAlias"); + } @Override public void upsertTagsAndAttributes( @@ -262,32 +696,83 @@ public void upsertTagsAndAttributes( Map tagsMap, Map attributesMap, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("upsertTagsAndAttributes"); + } @Override public void addAttributes(Map attributesMap, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("addAttributes"); + } @Override public void addTags(Map tagsMap, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("addTags"); + } @Override public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("dropTagsOrAttributes"); + } @Override public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("setTagsOrAttributesValue"); + } @Override public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) - throws MetadataException, IOException {} + throws MetadataException, IOException { + throw new UnsupportedOperationException("renameTagOrAttributeKey"); + } - @Override + // insert data + @Override // [iotdb/insert ] public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) throws MetadataException, IOException { - return null; + PartialPath devicePath = plan.getDevicePath(); + devicePath = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath( + storageGroupFullPath, devicePath.getFullPath())); + plan.setDevicePath(devicePath); + String[] measurementList = plan.getMeasurements(); + IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); + checkAlignedAndAutoCreateSeries(plan); + IMNode deviceMNode = getDeviceNode(devicePath); + IMeasurementMNode measurementMNode; + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (int i = 0; i < measurementList.length; i++) { + SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); + measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); + // check type is match + try { + SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); + } catch (DataTypeMismatchException mismatchException) { + if (!config.isEnablePartialInsert()) { + throw mismatchException; + } else { + // mark failed measurement + plan.markFailedMeasurementInsertion(i, mismatchException); + continue; + } + } + measurementMNodes[i] = measurementMNode; + } + plan.setDeviceID(deviceEntry.getDeviceID()); + plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); + return deviceMNode; + } + + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); } @Override @@ -297,48 +782,140 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( Function getDataType, boolean aligned) throws MetadataException { - return null; + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(getDataType.apply(i))); + + } else { + internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { + String[] measurementList = plan.getMeasurements(); + try { + if (plan.isAligned()) { + internalAlignedCreateTimeseries( + plan.getDevicePath(), + Arrays.asList(measurementList), + Arrays.asList(plan.getDataTypes())); + } else { + internalCreateTimeseries( + plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); + } + } catch (MetadataException e) { + if (!(e instanceof PathAlreadyExistException)) { + throw e; + } + } + } + + private void internalCreateTimeseries(PartialPath path, TSDataType dataType) + throws MetadataException { + createTimeseries( + path, + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor(), + Collections.emptyMap()); + } + + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, List measurements, List dataTypes) + throws MetadataException { + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + for (TSDataType dataType : dataTypes) { + encodings.add(getDefaultEncoding(dataType)); + compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); } @Override public Set getPathsSetTemplate(String templateName) throws MetadataException { - return null; + throw new UnsupportedOperationException("getPathsSetTemplate"); } @Override public Set getPathsUsingTemplate(String templateName) throws MetadataException { - return null; + throw new UnsupportedOperationException("getPathsUsingTemplate"); } @Override public boolean isTemplateAppendable(Template template, List measurements) throws MetadataException { - return false; + throw new UnsupportedOperationException("isTemplateAppendable"); } @Override - public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {} + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("setSchemaTemplate"); + } @Override - public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {} + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("unsetSchemaTemplate"); + } @Override - public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {} + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("setUsingSchemaTemplate"); + } @Override public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) - throws MetadataException {} + throws MetadataException { + throw new UnsupportedOperationException("activateSchemaTemplate"); + } @Override public List getPathsUsingTemplate(int templateId) throws MetadataException { - return null; + throw new UnsupportedOperationException("getPathsUsingTemplate"); } @Override public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { - return null; + throw new UnsupportedOperationException("getMNodeForTrigger"); } @Override - public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {} + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { + throw new UnsupportedOperationException("releaseMNodeAfterDropTrigger"); + } + + @Override + public String toString() { + return "TagSchemaRegion{" + + "storageGroupFullPath='" + + storageGroupFullPath + + '\'' + + ", schemaRegionId=" + + schemaRegionId + + ", schemaRegionDirPath='" + + schemaRegionDirPath + + '\'' + + '}'; + } } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java new file mode 100644 index 0000000000000..32475d650c560 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +public class AppendOnlyDeviceIDListFileManager { + + private static final Logger logger = + LoggerFactory.getLogger(AppendOnlyDeviceIDListFileManager.class); + + private static final String DEVICE_ID_LIST_FILE_NAME = "device_id.list"; + + private boolean isRecover; + + private String schemaDirPath; + + private File deviceIDSFile; + + private FileOutputStream outputStream; + + public AppendOnlyDeviceIDListFileManager(String schemaDirPath) { + this.schemaDirPath = schemaDirPath; + isRecover = true; + try { + initFile(); + outputStream = new FileOutputStream(deviceIDSFile, true); + } catch (IOException e) { + logger.error(e.getMessage()); + throw new IllegalArgumentException( + "can't initialize device id list file at " + deviceIDSFile); + } + } + + private void initFile() throws IOException { + File schemaDir = new File(schemaDirPath); + schemaDir.mkdirs(); + deviceIDSFile = new File(schemaDir, DEVICE_ID_LIST_FILE_NAME); + if (!deviceIDSFile.exists()) { + // create new file + deviceIDSFile.createNewFile(); + } + } + + public void serialize(String deviceID) { + try { + if (!isRecover) { + ReadWriteIOUtils.write(deviceID, outputStream); + } + } catch (IOException e) { + logger.error("failed to serialize device id: " + deviceID); + throw new IllegalArgumentException("can't serialize device id of " + deviceID); + } + } + + public void recover(DeviceIDList deviceIDList) { + logger.info("recover device id list using file {}", deviceIDSFile); + try (FileInputStream inputStream = new FileInputStream(deviceIDSFile)) { + while (inputStream.available() > 0) { + String deviceID = ReadWriteIOUtils.readString(inputStream); + deviceIDList.add(DeviceIDFactory.getInstance().getDeviceID(deviceID)); + } + } catch (IOException e) { + logger.info("device id list is incomplete, we will recover as much as we can."); + } + isRecover = false; + } + + @TestOnly + public void close() throws IOException { + try { + outputStream.close(); + } catch (IOException e) { + logger.error("close device id list file failed"); + throw e; + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java new file mode 100644 index 0000000000000..c8793d40fdfa4 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class DeviceIDList implements IDeviceIDList { + + private final List deviceIDS; + + private AppendOnlyDeviceIDListFileManager appendOnlyDeviceIDListFileManager; + + public DeviceIDList(String schemaDirPath) { + deviceIDS = new ArrayList<>(); + appendOnlyDeviceIDListFileManager = new AppendOnlyDeviceIDListFileManager(schemaDirPath); + recover(); + } + + public void recover() { + appendOnlyDeviceIDListFileManager.recover(this); + } + + @Override + public void add(IDeviceID deviceID) { + deviceIDS.add(deviceID); + appendOnlyDeviceIDListFileManager.serialize(deviceID.toStringID()); + } + + @Override + public IDeviceID get(int index) { + return deviceIDS.get(index); + } + + @Override + public int size() { + return deviceIDS.size(); + } + + @Override + public List getAllDeviceIDS() { + return new ArrayList<>(deviceIDS); + } + + @TestOnly + public void clear() throws IOException { + appendOnlyDeviceIDListFileManager.close(); + deviceIDS.clear(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java new file mode 100644 index 0000000000000..1a0d72571a8cd --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; + +import java.io.IOException; +import java.util.List; + +public interface IDeviceIDList { + + void add(IDeviceID deviceID); + + IDeviceID get(int index); + + int size(); + + List getAllDeviceIDS(); + + @TestOnly + void clear() throws IOException; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index ed62c3d2006d7..3ad63e48b8092 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -53,35 +53,39 @@ public class TagInvertedIndex implements ITagInvertedIndex { private static final TagSchemaConfig tagSchemaConfig = TagSchemaDescriptor.getInstance().getTagSchemaConfig(); - private final InsertionManager insertionManager; + private InsertionManager insertionManager; - private final DeletionManager deletionManager; + private DeletionManager deletionManager; - private final QueryManager queryManager; + private QueryManager queryManager; - private final WALManager walManager; + private WALManager walManager; - private final RecoverManager recoverManager; + private RecoverManager recoverManager; - private final int numOfDeviceIdsInMemTable; + private int numOfDeviceIdsInMemTable; - private final Map immutableMemTables; + private Map immutableMemTables; private MemTable workingMemTable; private int maxDeviceID; - public TagInvertedIndex(String schemaDirPath) throws IOException { - walManager = new WALManager(schemaDirPath); - insertionManager = new InsertionManager(walManager); - deletionManager = new DeletionManager(walManager); - recoverManager = new RecoverManager(walManager); - queryManager = new QueryManager(); - workingMemTable = new MemTable(MemTable.WORKING); - immutableMemTables = new HashMap<>(); - numOfDeviceIdsInMemTable = tagSchemaConfig.getNumOfDeviceIdsInMemTable(); - maxDeviceID = 0; - recover(); + public TagInvertedIndex(String schemaDirPath) { + try { + walManager = new WALManager(schemaDirPath); + insertionManager = new InsertionManager(walManager); + deletionManager = new DeletionManager(walManager); + recoverManager = new RecoverManager(walManager); + queryManager = new QueryManager(); + workingMemTable = new MemTable(MemTable.WORKING); + immutableMemTables = new HashMap<>(); + numOfDeviceIdsInMemTable = tagSchemaConfig.getNumOfDeviceIdsInMemTable(); + maxDeviceID = 0; + recover(); + } catch (IOException e) { + logger.error("create TagInvertedIndex fail", e); + } } public synchronized void recover() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index ec9fd74729565..7cc0015626030 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -24,7 +24,11 @@ import org.apache.iotdb.lsm.context.DeleteContext; import org.apache.iotdb.lsm.context.InsertContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class RecoverManager { + private static final Logger logger = LoggerFactory.getLogger(RecoverManager.class); private WALManager walManager; @@ -33,6 +37,7 @@ public RecoverManager(WALManager walManager) { } public void recover(ITagInvertedIndex tagInvertedIndex) { + logger.info("recover tagInvertedIndex"); while (true) { Context context = walManager.read(); switch (context.getType()) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java new file mode 100644 index 0000000000000..22ae4c3b58162 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +public class MeasurementPathUtils { + public static MeasurementPath generateMeasurementPath( + String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) + throws IllegalPathException { + MeasurementPath measurementPath = + new MeasurementPath( + devicePath, + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(isAligned); + return measurementPath; + } + + public static MeasurementPath generateMeasurementPath(DiskSchemaEntry diskSchemaEntry) + throws IllegalPathException { + MeasurementPath measurementPath = + new MeasurementPath( + new PartialPath(diskSchemaEntry.seriesKey), + new MeasurementSchema( + diskSchemaEntry.measurementName, + TSDataType.deserialize(diskSchemaEntry.type), + TSEncoding.deserialize(diskSchemaEntry.encoding), + CompressionType.deserialize(diskSchemaEntry.compressor))); + measurementPath.setUnderAlignedEntity(diskSchemaEntry.isAligned); + return measurementPath; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java new file mode 100644 index 0000000000000..bd2c3a4e9ab58 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import java.util.Map; +import java.util.TreeMap; + +public class PathTagConverterUtils { + + public static Map pathToTags(String storageGroupFullPath, String path) { + if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); + String devicePath = path.substring(storageGroupFullPath.length() + 1); + String[] tags = devicePath.split("\\."); + Map tagsMap = new TreeMap<>(); + for (int i = 0; i < tags.length; i += 2) { + tagsMap.put(tags[i], tags[i + 1]); + } + return tagsMap; + } + + public static String tagsToPath(String storageGroupFullPath, Map tags) { + StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); + for (String tagKey : tags.keySet()) { + stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); + } + return stringBuilder.toString(); + } + + public static String pathToTagsSortPath(String storageGroupFullPath, String path) { + return tagsToPath(storageGroupFullPath, pathToTags(storageGroupFullPath, path)); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java new file mode 100644 index 0000000000000..14f02a7dfd205 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; + +import java.util.HashMap; + +public class ShowTimeSeriesResultUtils { + public static ShowTimeSeriesResult generateShowTimeSeriesResult( + String sgName, String devicePath, String measurement, SchemaEntry schemaEntry) { + return new ShowTimeSeriesResult( + devicePath + "." + measurement, + null, + sgName, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>()); + } + + public static ShowTimeSeriesResult generateShowTimeSeriesResult( + String sgName, String timeSeriesPath, SchemaEntry schemaEntry) { + return new ShowTimeSeriesResult( + timeSeriesPath, + null, + sgName, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>()); + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java new file mode 100644 index 0000000000000..eddc31d2f9e08 --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class DeviceIDListTest { + + private IDeviceIDList deviceIDList; + + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testDeviceIDList"; + + private boolean isEnableIDTable; + + private String originalDeviceIDTransformationMethod; + + private String schemaDir; + + private String[] devicePaths = + new String[] { + storageGroupFullPath + ".a.b.c.d1", + storageGroupFullPath + ".a.b.c.d2", + storageGroupFullPath + ".a.b.c.d3", + storageGroupFullPath + ".a.b.c.d4", + storageGroupFullPath + ".a.b.d.d1", + storageGroupFullPath + ".a.b.d.d2", + storageGroupFullPath + ".a.b.e.d1", + storageGroupFullPath + ".a.b.f.d1", + storageGroupFullPath + ".a.b.g.d1", + storageGroupFullPath + ".a.b.h.d1", + }; + + @Before + public void setUp() throws Exception { + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); + originalDeviceIDTransformationMethod = + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + deviceIDList = new DeviceIDList(schemaRegionDirPath); + } + + @After + public void tearDown() throws Exception { + deviceIDList.clear(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); + IoTDBDescriptor.getInstance() + .getConfig() + .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); + } + + @Test + public void testAddandGetDeviceID() { + List deviceIDS = generateTestDeviceIDS(); + for (IDeviceID deviceID : deviceIDS) { + deviceIDList.add(deviceID); + System.out.println(deviceID.toStringID()); + } + for (int i = 0; i < deviceIDS.size(); i++) { + assertEquals(deviceIDS.get(i), deviceIDList.get(i)); + } + } + + @Test + public void testRecover() { + List deviceIDS = generateTestDeviceIDS(); + for (IDeviceID deviceID : deviceIDS) { + deviceIDList.add(deviceID); + System.out.println(deviceID.toStringID()); + } + deviceIDList = new DeviceIDList(schemaRegionDirPath); + for (int i = 0; i < deviceIDS.size(); i++) { + assertEquals(deviceIDS.get(i), deviceIDList.get(i)); + } + } + + private List generateTestDeviceIDS() { + List deviceIDS = new ArrayList<>(); + for (String devicePath : devicePaths) { + deviceIDS.add(DeviceIDFactory.getInstance().getDeviceID(devicePath)); + } + return deviceIDS; + } +} diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties index 38d0e3d1e3f40..a8d323200439d 100644 --- a/server/src/assembly/resources/conf/iotdb-datanode.properties +++ b/server/src/assembly/resources/conf/iotdb-datanode.properties @@ -1080,10 +1080,23 @@ timestamp_precision=ms # Datatype: float # group_by_fill_cache_size_in_mb=1.0 +#################### +### IDTable Configuration +#################### +# Datatype: boolean +# enable_id_table=false + +# Datatype: boolean +# enable_id_table_log_file=false + +# Choose the device id transformation method. The value could be Plain and SHA256. If the provided value doesn't match any pre-defined value, Plain will be used as default. +# Datatype: string +# device_id_transformation_method=Plain + #################### ### Schema Engine Configuration #################### -# Choose the mode of schema engine. The value could be Memory,Schema_File and Rocksdb_based. If the provided value doesn't match any pre-defined value, Memory mode will be used as default. +# Choose the mode of schema engine. The value could be Memory,Schema_File,Rocksdb_based and Tag. If the provided value doesn't match any pre-defined value, Memory mode will be used as default. # Datatype: string # schema_engine_mode=Memory diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java index e0e21ddbfaa69..f6bd30867cf18 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java @@ -74,6 +74,10 @@ public DiskSchemaEntry( this.isAligned = isAligned; } + public String getDevicePath() { + return seriesKey.substring(0, seriesKey.length() - measurementName.length() - 1); + } + public int serialize(OutputStream outputStream) throws IOException { int byteLen = 0; byteLen += ReadWriteIOUtils.write(deviceID, outputStream); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java index 7b3d243242dbd..c6762738695ca 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java @@ -35,7 +35,6 @@ import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; import org.apache.iotdb.db.metadata.mtree.ConfigMTree; import org.apache.iotdb.db.metadata.rescon.SchemaResourceManager; -import org.apache.iotdb.db.metadata.schemaregion.tagschemaregion.MockTagSchemaRegion; import org.apache.iotdb.db.metadata.visitor.SchemaExecutionVisitor; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.external.api.ISeriesNumerLimiter; @@ -326,8 +325,9 @@ private ISchemaRegion createSchemaRegionWithoutExistenceCheck( break; case Tag: schemaRegion = - new MockTagSchemaRegion( - storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter); + new TagSchemaRegionLoader() + .loadTagSchemaRegion( + storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter); break; default: throw new UnsupportedOperationException( diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java index 46a8056d05b89..694ffe7929d61 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,16 +40,14 @@ public class TagSchemaRegionLoader { private static URLClassLoader urlClassLoader = null; private static final String TAG_SCHEMA_REGION_CLASS_NAME = "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaRegion"; - private static final String TAG_SCHEMA_CONF_DESCRIPTOR_CLASS_NAME = - "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor"; private static final String LIB_PATH = ".." + File.separator + "lib" + File.separator + "tag-schema-region" + File.separator; public TagSchemaRegionLoader() {} /** - * Load the jar files for RSchemaRegion and create an instance of it. The jar files should be - * located in "../lib/rschema-region". If jar files cannot be found, the function will return + * Load the jar files for TagSchemaRegion and create an instance of it. The jar files should be + * located in "../lib/tag-schema-region". If jar files cannot be found, the function will return * null. * * @param storageGroup @@ -56,25 +55,26 @@ public TagSchemaRegionLoader() {} * @param node * @return */ - public ISchemaRegion loadRSchemaRegion( - PartialPath storageGroup, SchemaRegionId schemaRegionId, IStorageGroupMNode node) { + public ISchemaRegion loadTagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode node, + ISeriesNumerLimiter seriesNumerLimiter) { ISchemaRegion region = null; - LOGGER.info("Creating instance for schema-engine-rocksdb"); + LOGGER.info("Creating instance for schema-engine-tag"); try { - loadRSchemaRegionJar(); + loadTagSchemaRegionJar(); Class classForTagSchemaRegion = urlClassLoader.loadClass(TAG_SCHEMA_REGION_CLASS_NAME); - Class classForTagSchemaDescriptor = - urlClassLoader.loadClass(TAG_SCHEMA_CONF_DESCRIPTOR_CLASS_NAME); Constructor constructor = classForTagSchemaRegion.getConstructor( PartialPath.class, SchemaRegionId.class, IStorageGroupMNode.class, - classForTagSchemaDescriptor); - Object rSchemaLoader = classForTagSchemaDescriptor.getConstructor().newInstance(); + ISeriesNumerLimiter.class); + region = (ISchemaRegion) - constructor.newInstance(storageGroup, schemaRegionId, node, rSchemaLoader); + constructor.newInstance(storageGroup, schemaRegionId, node, seriesNumerLimiter); } catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException @@ -82,18 +82,18 @@ public ISchemaRegion loadRSchemaRegion( | IllegalAccessException | MalformedURLException | RuntimeException e) { - LOGGER.error("Cannot initialize RSchemaRegion", e); + LOGGER.error("Cannot initialize TagSchemaRegion", e); return null; } return region; } /** - * Load the jar files for rocksdb and RSchemaRegion. The jar files should be located in directory - * "../lib/rschema-region". If the jar files have been loaded, it will do nothing. + * Load the jar files for TagSchemaRegion. The jar files should be located in directory + * "../lib/tag-schema-region". If the jar files have been loaded, it will do nothing. */ - private void loadRSchemaRegionJar() throws MalformedURLException { - LOGGER.info("Loading jar for schema-engine-rocksdb"); + private void loadTagSchemaRegionJar() throws MalformedURLException { + LOGGER.info("Loading jar for schema-engine-tag"); if (urlClassLoader == null) { File[] jars = new File(LIB_PATH).listFiles(); if (jars == null) { @@ -106,6 +106,7 @@ private void loadRSchemaRegionJar() throws MalformedURLException { dependentJars.add(new URL("file:" + jar.getAbsolutePath())); } } + LOGGER.info("load jars: " + dependentJars); urlClassLoader = new URLClassLoader(dependentJars.toArray(new URL[] {})); } } From 33dc78c94323bc699acbf120892731c23ca69452 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 22 Sep 2022 17:36:31 +0800 Subject: [PATCH 22/53] add SchemaTreeVisitorFactory --- .../common/schematree/ClusterSchemaTree.java | 68 ++--------- .../visitor/SchemaTreeVisitorFactory.java | 112 ++++++++++++++++++ 2 files changed, 120 insertions(+), 60 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java index 9974b982c11ae..112a9137f5e9b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java @@ -22,15 +22,14 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.path.MeasurementPath; -import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaEntityNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaInternalNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaMeasurementNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaNode; import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeDeviceVisitor; import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeMeasurementVisitor; +import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeVisitorFactory; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; @@ -79,49 +78,8 @@ public Pair, Integer> searchMeasurementPaths( @Override public Pair, Integer> searchMeasurementPaths(PartialPath pathPattern) { - SchemaTreeMeasurementVisitor visitor; - switch (SchemaEngineMode.valueOf( - IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { - case Memory: - case Schema_File: - visitor = - new SchemaTreeMeasurementVisitor( - root, - pathPattern, - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); - break; - case Tag: - if (pathPattern.getFullPath().contains(".**")) { - String measurement = pathPattern.getMeasurement(); - visitor = - new SchemaTreeMeasurementVisitor( - root, - ALL_MATCH_PATTERN.concatNode(measurement), - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); - } else { - visitor = - new SchemaTreeMeasurementVisitor( - root, - pathPattern, - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); - } - break; - default: - visitor = - new SchemaTreeMeasurementVisitor( - root, - ALL_MATCH_PATTERN, - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); - break; - } + SchemaTreeMeasurementVisitor visitor = + SchemaTreeVisitorFactory.getInstance().getSchemaTreeMeasurementVisitor(root, pathPattern); return new Pair<>(visitor.getAllResult(), visitor.getNextOffset()); } @@ -138,26 +96,16 @@ public List getAllMeasurement() { */ @Override public List getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) { - SchemaTreeDeviceVisitor visitor = new SchemaTreeDeviceVisitor(root, pathPattern, isPrefixMatch); + SchemaTreeDeviceVisitor visitor = + SchemaTreeVisitorFactory.getInstance() + .getSchemaTreeDeviceVisitor(root, pathPattern, isPrefixMatch); return visitor.getAllResult(); } @Override public List getMatchedDevices(PartialPath pathPattern) { - SchemaTreeDeviceVisitor visitor; - switch (SchemaEngineMode.valueOf( - IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { - case Memory: - case Schema_File: - visitor = new SchemaTreeDeviceVisitor(root, pathPattern, false); - break; - case Tag: - visitor = new SchemaTreeDeviceVisitor(root, ALL_MATCH_PATTERN, false); - break; - default: - visitor = new SchemaTreeDeviceVisitor(root, ALL_MATCH_PATTERN, false); - break; - } + SchemaTreeDeviceVisitor visitor = + SchemaTreeVisitorFactory.getInstance().getSchemaTreeDeviceVisitor(root, pathPattern, false); return visitor.getAllResult(); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java new file mode 100644 index 0000000000000..cc669ac6f7170 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.common.schematree.visitor; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; +import org.apache.iotdb.db.mpp.common.schematree.node.SchemaNode; + +import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_MATCH_PATTERN; + +/** + * Generate corresponding SchemaTreeVisitor for different types of schema regions, such as Memory, + * Schema_File, Rocksdb_based, and Tag + */ +public class SchemaTreeVisitorFactory { + + private SchemaTreeVisitorFactory() {} + + public static SchemaTreeVisitorFactory getInstance() { + return SchemaTreeVisitorFactoryHolder.INSTANCE; + } + + /** + * generate corresponding SchemaTreeDeviceVisitor for different types of schema regions + * + * @param schemaNode schema node + * @param pathPattern can be a pattern or a full path of timeseries. + * @param isPrefixMatch + * @return schemaTreeDeviceVisitor + */ + public SchemaTreeDeviceVisitor getSchemaTreeDeviceVisitor( + SchemaNode schemaNode, PartialPath pathPattern, boolean isPrefixMatch) { + SchemaTreeDeviceVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = new SchemaTreeDeviceVisitor(schemaNode, pathPattern, isPrefixMatch); + break; + default: + visitor = new SchemaTreeDeviceVisitor(schemaNode, ALL_MATCH_PATTERN, isPrefixMatch); + break; + } + return visitor; + } + + /** + * generate corresponding SchemaTreeMeasurementVisitor for different types of schema regions + * + * @param schemaNode schema node + * @param pathPattern can be a pattern or a full path of timeseries. + * @return schemaTreeMeasurementVisitor + */ + public SchemaTreeMeasurementVisitor getSchemaTreeMeasurementVisitor( + SchemaNode schemaNode, PartialPath pathPattern) { + SchemaTreeMeasurementVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = generateSchemaTreeMeasurementVisitor(schemaNode, pathPattern); + break; + case Tag: + if (pathPattern.getFullPath().contains(".**")) { + visitor = + generateSchemaTreeMeasurementVisitor( + schemaNode, ALL_MATCH_PATTERN.concatNode(pathPattern.getMeasurement())); + } else { + visitor = generateSchemaTreeMeasurementVisitor(schemaNode, pathPattern); + } + break; + default: + visitor = generateSchemaTreeMeasurementVisitor(schemaNode, ALL_MATCH_PATTERN); + break; + } + return visitor; + } + + private SchemaTreeMeasurementVisitor generateSchemaTreeMeasurementVisitor( + SchemaNode schemaNode, PartialPath pathPattern) { + return new SchemaTreeMeasurementVisitor( + schemaNode, + pathPattern, + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + } + + private static class SchemaTreeVisitorFactoryHolder { + private static final SchemaTreeVisitorFactory INSTANCE = new SchemaTreeVisitorFactory(); + + private SchemaTreeVisitorFactoryHolder() {} + } +} From f08f7a6120151f7c71ec45316499e224d8c5cb88 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 22 Sep 2022 20:33:05 +0800 Subject: [PATCH 23/53] add SchemaTreeVisitorFactory --- .../tagschemaregion/MockTagSchemaRegion.java | 927 ------------------ 1 file changed, 927 deletions(-) delete mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java deleted file mode 100644 index e4122375469c5..0000000000000 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/tagschemaregion/MockTagSchemaRegion.java +++ /dev/null @@ -1,927 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.metadata.schemaregion.tagschemaregion; - -import org.apache.iotdb.common.rpc.thrift.TSchemaNode; -import org.apache.iotdb.commons.consensus.SchemaRegionId; -import org.apache.iotdb.commons.exception.MetadataException; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; -import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; -import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; -import org.apache.iotdb.db.exception.metadata.PathNotExistException; -import org.apache.iotdb.db.metadata.LocalSchemaProcessor; -import org.apache.iotdb.db.metadata.idtable.IDTable; -import org.apache.iotdb.db.metadata.idtable.IDTableManager; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; -import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; -import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; -import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; -import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; -import org.apache.iotdb.db.metadata.mnode.EntityMNode; -import org.apache.iotdb.db.metadata.mnode.IMNode; -import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; -import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; -import org.apache.iotdb.db.metadata.path.MeasurementPath; -import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; -import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; -import org.apache.iotdb.db.metadata.template.Template; -import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; -import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; -import org.apache.iotdb.db.qp.physical.crud.InsertPlan; -import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; -import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; -import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; -import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; -import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; -import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; -import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; -import org.apache.iotdb.db.query.context.QueryContext; -import org.apache.iotdb.db.query.dataset.ShowDevicesResult; -import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; -import org.apache.iotdb.external.api.ISeriesNumerLimiter; -import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.utils.Pair; -import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; - -import org.jetbrains.annotations.NotNull; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; - -import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; - -public class MockTagSchemaRegion implements ISchemaRegion { - - protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private final String TAIL = ".**"; - private final IStorageGroupMNode storageGroupMNode; - private String storageGroupFullPath; - private SchemaRegionId schemaRegionId; - - private Map>> tagInvertedIndex; - - private List deviceIDS; - - private IDTable idTable; - - private final ISeriesNumerLimiter seriesNumerLimiter; - - public MockTagSchemaRegion( - PartialPath storageGroup, - SchemaRegionId schemaRegionId, - IStorageGroupMNode storageGroupMNode, - ISeriesNumerLimiter seriesNumerLimiter) - throws MetadataException { - - storageGroupFullPath = storageGroup.getFullPath(); - this.schemaRegionId = schemaRegionId; - this.storageGroupMNode = storageGroupMNode; - this.deviceIDS = new ArrayList<>(); - this.seriesNumerLimiter = seriesNumerLimiter; - tagInvertedIndex = new ConcurrentHashMap<>(); - idTable = IDTableManager.getInstance().getIDTable(storageGroup); - init(); - } - - @NotNull - private Map pathToTags(String path) { - if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); - String devicePath = path.substring(storageGroupFullPath.length() + 1); - String[] tags = devicePath.split("\\."); - Map tagsMap = new TreeMap<>(); - for (int i = 0; i < tags.length; i += 2) { - tagsMap.put(tags[i], tags[i + 1]); - } - return tagsMap; - } - - public String tagsToPath(Map tags) { - StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); - for (String tagKey : tags.keySet()) { - stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); - } - return stringBuilder.toString(); - } - - @Override - public void init() throws MetadataException { - if (!config.isEnableIDTableLogFile() - && config.getDeviceIDTransformationMethod().equals("SHA256")) { - throw new MetadataException( - "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); - } - } - - @Override - public void clear() { - return; - } - - @Override - public void forceMlog() { - return; - } - - @Override - public SchemaRegionId getSchemaRegionId() { - return schemaRegionId; - } - - @Override - public String getStorageGroupFullPath() { - return storageGroupFullPath; - } - - @Override - public void deleteSchemaRegion() throws MetadataException { - return; - } - - @Override - public boolean createSnapshot(File snapshotDir) { - return false; - } - - @Override - public void loadSnapshot(File latestSnapshotRootDir) { - return; - } - - private void createTagInvertedIndex(PartialPath devicePath) { - IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); - Map tagsMap = pathToTags(devicePath.getFullPath()); - - deviceIDS.add(deviceID); - - for (String tagkey : tagsMap.keySet()) { - String tagValue = tagsMap.get(tagkey); - Map> tagkeyMap = - tagInvertedIndex.computeIfAbsent(tagkey, key -> new HashMap<>()); - List ids = tagkeyMap.computeIfAbsent(tagValue, key -> new ArrayList<>()); - ids.add(deviceIDS.size() - 1); - } - } - - private void createTimeseries( - PartialPath path, - TSDataType dataType, - TSEncoding encoding, - CompressionType compressor, - Map props) - throws MetadataException { - createTimeseries( - new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); - } - - private void createAlignedTimeSeries( - PartialPath prefixPath, - List measurements, - List dataTypes, - List encodings, - List compressors) - throws MetadataException { - createAlignedTimeSeries( - new CreateAlignedTimeSeriesPlan( - prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); - } - - @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] - public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { - PartialPath devicePath = plan.getPath().getDevicePath(); - Map tags = pathToTags(devicePath.getFullPath()); - PartialPath path = new PartialPath(tagsToPath(tags) + "." + plan.getPath().getMeasurement()); - plan.setPath(path); - devicePath = plan.getPath().getDevicePath(); - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); - if (deviceEntry != null) { - if (deviceEntry.isAligned()) { - throw new AlignedTimeseriesException( - "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", - devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); - } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { - throw new PathAlreadyExistException( - devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); - } - } - idTable.createTimeseries(plan); - if (deviceEntry == null) { - createTagInvertedIndex(devicePath); - } - } - - @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] - public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { - PartialPath devicePath = plan.getPrefixPath(); - Map tags = pathToTags(devicePath.getFullPath()); - PartialPath path = new PartialPath(tagsToPath(tags)); - plan.setPrefixPath(path); - devicePath = plan.getPrefixPath(); - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); - if (deviceEntry != null) { - if (!deviceEntry.isAligned()) { - throw new AlignedTimeseriesException( - "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", - devicePath.getFullPath()); - } else { - List measurements = plan.getMeasurements(); - List dataTypes = plan.getDataTypes(); - List encodings = plan.getEncodings(); - List compressors = plan.getCompressors(); - - List tmpMeasurements = new LinkedList<>(); - List tmpDataTypes = new LinkedList<>(); - List tmpEncodings = new LinkedList<>(); - List tmpCompressors = new LinkedList<>(); - for (int i = 0; i < measurements.size(); i++) { - String measurement = measurements.get(i); - if (!deviceEntry.getMeasurementMap().containsKey(measurement)) { - tmpMeasurements.add(measurements.get(i)); - tmpDataTypes.add(dataTypes.get(i)); - tmpEncodings.add(encodings.get(i)); - tmpCompressors.add(compressors.get(i)); - } - } - if (tmpMeasurements.size() == 0) - throw new PathAlreadyExistException(devicePath.getFullPath()); - plan.setMeasurements(tmpMeasurements); - plan.setDataTypes(tmpDataTypes); - plan.setEncodings(tmpEncodings); - plan.setCompressors(tmpCompressors); - } - } - idTable.createAlignedTimeseries(plan); - if (deviceEntry == null) { - createTagInvertedIndex(devicePath); - } - } - - @Override - public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { - return 0; - } - - @Override - public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} - - @Override - public List fetchSchemaBlackList(PathPatternTree patternTree) - throws MetadataException { - return null; - } - - @Override - public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} - - @Override - public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public boolean isPathExist(PartialPath path) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - int res = 0; - List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); - for (IDeviceID deviceID : deviceIDs) { - res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); - } - return res; - } - - @Override - public int getAllTimeseriesCount( - PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) - throws MetadataException { - return 0; - } - - @Override - public int getAllTimeseriesCount( - PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) - throws MetadataException { - return 0; - } - - @Override - public Map getMeasurementCountGroupByLevel( - PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Map getMeasurementCountGroupByLevel( - PartialPath pathPattern, - int level, - boolean isPrefixMatch, - String key, - String value, - boolean isContains) - throws MetadataException { - return null; - } - - @Override - public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { - return deviceIDS.size(); - } else { - return getDeviceIDsByInvertedIndex(pathPattern).size(); - } - } - - @Override - public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public List getNodesListInGivenLevel( - PartialPath pathPattern, - int nodeLevel, - boolean isPrefixMatch, - LocalSchemaProcessor.StorageGroupFilter filter) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Set getChildNodePathInNextLevel(PartialPath pathPattern) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] - public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); - Set res = new HashSet<>(); - String devicePath = pathPattern.getFullPath(); - if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); - if (deviceEntry != null) { - res.add(pathPattern); - } - return res; - } - for (IDeviceID deviceID : deviceIDs) { - if (deviceID instanceof SHA256DeviceID) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); - Map map = deviceEntry.getMeasurementMap(); - for (String m : map.keySet()) { - SchemaEntry schemaEntry = map.get(m); - List schemaEntries = new ArrayList<>(); - schemaEntries.add(schemaEntry); - List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); - DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); - res.add( - new PartialPath( - diskSchemaEntry.seriesKey.substring( - 0, - diskSchemaEntry.seriesKey.length() - - diskSchemaEntry.measurementName.length() - - 1))); - break; - } - } else { - res.add(new PartialPath(deviceID.toStringID())); - } - } - return res; - } - - @Override - public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select - // .. groupby level] - public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) - throws MetadataException { - PartialPath devicePath = pathPattern.getDevicePath(); - // 批量查询.路径以".**"结尾,如： - // root.sg.tag1.a.** - // root.sg.tagx.c.tag2.v.** - // 点查询.路径不以".**",直接走IDTable,精确查询 - if (devicePath.getFullPath().endsWith(TAIL)) { - return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); - } else { - return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); - } - } - - private List getMeasurementPathsWithPointQuery( - PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { - List res = new LinkedList<>(); - String path = devicePath.getFullPath(); - Map tags = pathToTags(path); - path = tagsToPath(tags); - DeviceEntry deviceEntry = idTable.getDeviceEntry(path); - if (deviceEntry == null) return res; - Map schemaMap = deviceEntry.getMeasurementMap(); - for (String measurement : schemaMap.keySet()) { - SchemaEntry schemaEntry = schemaMap.get(measurement); - MeasurementPath measurementPath = - new MeasurementPath( - path, - measurement, - new MeasurementSchema( - measurement, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType())); - measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); - res.add(measurementPath); - } - - return res; - } - - private List getMeasurementPathsWithBatchQuery( - PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { - List res = new LinkedList<>(); - List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); - for (IDeviceID deviceID : deviceIDs) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); - Map schemaMap = deviceEntry.getMeasurementMap(); - if (deviceID instanceof SHA256DeviceID) { - for (String measurement : schemaMap.keySet()) { - SchemaEntry schemaEntry = schemaMap.get(measurement); - List schemaEntries = new ArrayList<>(); - schemaEntries.add(schemaEntry); - List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); - DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); - MeasurementPath measurementPath = - new MeasurementPath( - new PartialPath(diskSchemaEntry.seriesKey), - new MeasurementSchema( - measurement, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType())); - measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); - res.add(measurementPath); - } - } else { - for (String measurement : schemaMap.keySet()) { - SchemaEntry schemaEntry = schemaMap.get(measurement); - MeasurementPath measurementPath = - new MeasurementPath( - deviceID.toStringID(), - measurement, - new MeasurementSchema( - measurement, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType())); - measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); - res.add(measurementPath); - } - } - } - return res; - } - - // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] - @Override - public Pair, Integer> getMeasurementPathsWithAlias( - PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) - throws MetadataException { - List res = getMeasurementPaths(pathPattern, isPrefixMatch); - Pair, Integer> result = new Pair<>(res, 0); - return result; - } - - @Override - public List fetchSchema( - PartialPath pathPattern, Map templateMap) throws MetadataException { - return null; - } - - // show 时间序列 - @Override // [iotdb/show timeseries] - public Pair, Integer> showTimeseries( - ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { - List res = new ArrayList<>(); - Pair, Integer> result = new Pair<>(res, 0); - String path = plan.getPath().getFullPath(); - if (!path.endsWith(TAIL)) { - Map tags = pathToTags(path); - path = tagsToPath(tags); - DeviceEntry deviceEntry = idTable.getDeviceEntry(path); - if (deviceEntry != null) { - Map measurementMap = deviceEntry.getMeasurementMap(); - for (String m : measurementMap.keySet()) { - SchemaEntry schemaEntry = measurementMap.get(m); - res.add( - new ShowTimeSeriesResult( - path + "." + m, - "null", - storageGroupFullPath, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType(), - schemaEntry.getLastTime(), - new HashMap<>(), - new HashMap<>())); - } - } - return result; - } - List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); - for (IDeviceID deviceID : deviceIDs) { - getTimeSeriesResultOfDeviceFromIDTable(res, deviceID); - } - return result; - } - - private List getDeviceIdFromInvertedIndex(PartialPath devicePath) - throws MetadataException { - String path = devicePath.getFullPath(); - if (path.endsWith(TAIL)) { - path = path.substring(0, path.length() - TAIL.length()); - devicePath = new PartialPath(path); - } - if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { - return deviceIDS; - } else { - List res = new LinkedList<>(); - List ids = getDeviceIDsByInvertedIndex(devicePath); - if (ids.size() > 0) { - for (int id : ids) { - res.add(deviceIDS.get(id)); - } - } - return res; - } - } - - private void getTimeSeriesResultOfDeviceFromIDTable( - List res, IDeviceID deviceID) { - Map measurementMap = - idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); - if (deviceID instanceof SHA256DeviceID) { - for (String m : measurementMap.keySet()) { - SchemaEntry schemaEntry = measurementMap.get(m); - List schemaEntries = new ArrayList<>(); - schemaEntries.add(schemaEntry); - List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); - DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); - res.add( - new ShowTimeSeriesResult( - diskSchemaEntry.seriesKey, - "null", - storageGroupFullPath, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType(), - schemaEntry.getLastTime(), - new HashMap<>(), - new HashMap<>())); - } - } else { - for (String m : measurementMap.keySet()) { - SchemaEntry schemaEntry = measurementMap.get(m); - res.add( - new ShowTimeSeriesResult( - deviceID.toStringID() + "." + m, - "null", - storageGroupFullPath, - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType(), - schemaEntry.getLastTime(), - new HashMap<>(), - new HashMap<>())); - } - } - } - - private List getDeviceIDsByInvertedIndex(PartialPath path) { - Map tags = pathToTags(path.getFullPath()); - List idsCollection = new ArrayList<>(tags.keySet().size()); - for (String tagKey : tags.keySet()) { - if (!tagInvertedIndex.containsKey(tagKey) - || !tagInvertedIndex.get(tagKey).containsKey(tags.get(tagKey))) { - return new ArrayList<>(); - } - List ids = tagInvertedIndex.get(tagKey).get(tags.get(tagKey)); - idsCollection.add(new ArrayList(ids)); - } - if (idsCollection.size() == 0) return new ArrayList<>(); - List ids = idsCollection.get(0); - for (int i = 1; i < idsCollection.size(); i++) { - List list = idsCollection.get(i); - ids.retainAll(list); - } - return ids; - } - - @Override - public List getAllMeasurementByDevicePath(PartialPath devicePath) - throws PathNotExistException { - throw new UnsupportedOperationException(""); - } - - @Override - public IMNode getDeviceNode(PartialPath path) throws MetadataException { - DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); - if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); - return new EntityMNode(storageGroupMNode, path.getFullPath()); - } - - @Override - public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void upsertTagsAndAttributes( - String alias, - Map tagsMap, - Map attributesMap, - PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void addAttributes(Map attributesMap, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void addTags(Map tagsMap, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - @Override - public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) - throws MetadataException, IOException { - throw new UnsupportedOperationException(""); - } - - // insert data - @Override // [iotdb/insert ] - public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) - throws MetadataException, IOException { - PartialPath devicePath = plan.getDevicePath(); - Map tags = pathToTags(devicePath.getFullPath()); - devicePath = new PartialPath(tagsToPath(tags)); - plan.setDevicePath(devicePath); - String[] measurementList = plan.getMeasurements(); - IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); - checkAlignedAndAutoCreateSeries(plan); - IMNode deviceMNode = getDeviceNode(devicePath); - IMeasurementMNode measurementMNode; - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); - Map schemaMap = deviceEntry.getMeasurementMap(); - for (int i = 0; i < measurementList.length; i++) { - SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); - // measurementMNode = - // new MeasurementMNode( - // deviceMNode, - // measurementList[i], - // new MeasurementSchema( - // measurementList[i], - // schemaEntry.getTSDataType(), - // schemaEntry.getTSEncoding(), - // schemaEntry.getCompressionType()), - // null); - measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); - // check type is match - try { - SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); - } catch (DataTypeMismatchException mismatchException) { - if (!config.isEnablePartialInsert()) { - throw mismatchException; - } else { - // mark failed measurement - plan.markFailedMeasurementInsertion(i, mismatchException); - continue; - } - } - measurementMNodes[i] = measurementMNode; - } - plan.setDeviceID(deviceEntry.getDeviceID()); - plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); - return deviceMNode; - } - - private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); - if (deviceEntry == null) return null; - return deviceEntry.getSchemaEntry(measurementName); - } - - @Override - public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( - PartialPath devicePath, - String[] measurements, - Function getDataType, - boolean aligned) - throws MetadataException { - List measurementSchemaInfoList = new ArrayList<>(measurements.length); - for (int i = 0; i < measurements.length; i++) { - SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); - if (schemaEntry == null) { - if (config.isAutoCreateSchemaEnabled()) { - if (aligned) { - internalAlignedCreateTimeseries( - devicePath, - Collections.singletonList(measurements[i]), - Collections.singletonList(getDataType.apply(i))); - - } else { - internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); - } - } - schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); - } - measurementSchemaInfoList.add( - new MeasurementSchemaInfo( - measurements[i], - new MeasurementSchema( - measurements[i], - schemaEntry.getTSDataType(), - schemaEntry.getTSEncoding(), - schemaEntry.getCompressionType()), - null)); - } - return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); - } - - private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { - String[] measurementList = plan.getMeasurements(); - try { - if (plan.isAligned()) { - internalAlignedCreateTimeseries( - plan.getDevicePath(), - Arrays.asList(measurementList), - Arrays.asList(plan.getDataTypes())); - } else { - internalCreateTimeseries( - plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); - } - } catch (MetadataException e) { - if (!(e instanceof PathAlreadyExistException)) { - throw e; - } - } - } - - private void internalCreateTimeseries(PartialPath path, TSDataType dataType) - throws MetadataException { - createTimeseries( - path, - dataType, - getDefaultEncoding(dataType), - TSFileDescriptor.getInstance().getConfig().getCompressor(), - Collections.emptyMap()); - } - - private void internalAlignedCreateTimeseries( - PartialPath prefixPath, List measurements, List dataTypes) - throws MetadataException { - List encodings = new ArrayList<>(); - List compressors = new ArrayList<>(); - for (TSDataType dataType : dataTypes) { - encodings.add(getDefaultEncoding(dataType)); - compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); - } - createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); - } - - @Override - public Set getPathsSetTemplate(String templateName) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public Set getPathsUsingTemplate(String templateName) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public boolean isTemplateAppendable(Template template, List measurements) - throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) - throws MetadataException {} - - @Override - public List getPathsUsingTemplate(int templateId) throws MetadataException { - return null; - } - - @Override - public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { - throw new UnsupportedOperationException(""); - } - - @Override - public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { - throw new UnsupportedOperationException(""); - } -} From 88399fc6148f9aa92ffe2d5038cff717de6bd8b6 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Thu, 22 Sep 2022 21:39:16 +0800 Subject: [PATCH 24/53] mod getDeviceSchemaInfoWithAutoCreate --- .../tagSchemaRegion/MockTagSchemaRegion.java | 87 ++++++++++++++++--- .../tagSchemaRegion/TagSchemaRegion.java | 55 ++++++++++-- 2 files changed, 122 insertions(+), 20 deletions(-) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java index bec4ba83947d7..bb1fcce53987b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -762,16 +762,6 @@ public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) Map schemaMap = deviceEntry.getMeasurementMap(); for (int i = 0; i < measurementList.length; i++) { SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); - // measurementMNode = - // new MeasurementMNode( - // deviceMNode, - // measurementList[i], - // new MeasurementSchema( - // measurementList[i], - // schemaEntry.getTSDataType(), - // schemaEntry.getTSEncoding(), - // schemaEntry.getCompressionType()), - // null); measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); // check type is match try { @@ -792,14 +782,63 @@ public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) return deviceMNode; } + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + TSDataType dataType = getDataType.apply(i); + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(dataType), + Collections.singletonList( + encodings[i] == null ? getDefaultEncoding(dataType) : encodings[i]), + Collections.singletonList( + compressionTypes[i] == null + ? TSFileDescriptor.getInstance().getConfig().getCompressor() + : compressionTypes[i])); + + } else { + internalCreateTimeseries( + devicePath.concatNode(measurements[i]), + getDataType.apply(i), + encodings[i], + compressionTypes[i]); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); if (deviceEntry == null) return null; return deviceEntry.getSchemaEntry(measurementName); } - @Override - public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + private DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( PartialPath devicePath, String[] measurements, Function getDataType, @@ -864,6 +903,19 @@ private void internalCreateTimeseries(PartialPath path, TSDataType dataType) Collections.emptyMap()); } + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries( + PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor) + throws MetadataException { + if (encoding == null) { + encoding = getDefaultEncoding(dataType); + } + if (compressor == null) { + compressor = TSFileDescriptor.getInstance().getConfig().getCompressor(); + } + createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap()); + } + private void internalAlignedCreateTimeseries( PartialPath prefixPath, List measurements, List dataTypes) throws MetadataException { @@ -876,6 +928,17 @@ private void internalAlignedCreateTimeseries( createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); } + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + @Override public Set getPathsSetTemplate(String templateName) throws MetadataException { throw new UnsupportedOperationException(""); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index c32466cd346f0..7ad604a9c218c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -769,17 +769,13 @@ public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) return deviceMNode; } - private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { - DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); - if (deviceEntry == null) return null; - return deviceEntry.getSchemaEntry(measurementName); - } - @Override public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( PartialPath devicePath, String[] measurements, Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, boolean aligned) throws MetadataException { List measurementSchemaInfoList = new ArrayList<>(measurements.length); @@ -788,13 +784,24 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( if (schemaEntry == null) { if (config.isAutoCreateSchemaEnabled()) { if (aligned) { + TSDataType dataType = getDataType.apply(i); internalAlignedCreateTimeseries( devicePath, Collections.singletonList(measurements[i]), - Collections.singletonList(getDataType.apply(i))); + Collections.singletonList(dataType), + Collections.singletonList( + encodings[i] == null ? getDefaultEncoding(dataType) : encodings[i]), + Collections.singletonList( + compressionTypes[i] == null + ? TSFileDescriptor.getInstance().getConfig().getCompressor() + : compressionTypes[i])); } else { - internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); + internalCreateTimeseries( + devicePath.concatNode(measurements[i]), + getDataType.apply(i), + encodings[i], + compressionTypes[i]); } } schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); @@ -812,6 +819,12 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); } + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); + } + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { String[] measurementList = plan.getMeasurements(); try { @@ -831,6 +844,7 @@ private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataExc } } + /** create timeseries ignoring PathAlreadyExistException */ private void internalCreateTimeseries(PartialPath path, TSDataType dataType) throws MetadataException { createTimeseries( @@ -841,6 +855,20 @@ private void internalCreateTimeseries(PartialPath path, TSDataType dataType) Collections.emptyMap()); } + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries( + PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor) + throws MetadataException { + if (encoding == null) { + encoding = getDefaultEncoding(dataType); + } + if (compressor == null) { + compressor = TSFileDescriptor.getInstance().getConfig().getCompressor(); + } + createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap()); + } + + /** create aligned timeseries ignoring PathAlreadyExistException */ private void internalAlignedCreateTimeseries( PartialPath prefixPath, List measurements, List dataTypes) throws MetadataException { @@ -853,6 +881,17 @@ private void internalAlignedCreateTimeseries( createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); } + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + @Override public Set getPathsSetTemplate(String templateName) throws MetadataException { throw new UnsupportedOperationException("getPathsSetTemplate"); From 4838571974860f732f4d3cdcf45b11bb8c8bfe7e Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 10:53:09 +0800 Subject: [PATCH 25/53] mod Licensed --- lsm/pom.xml | 28 +++++++++++-------- .../org/apache/iotdb/lsm/context/Context.java | 7 ----- .../iotdb/lsm/example/MemTableManager.java | 4 --- .../iotdb/lsm/strategy/AccessStrategy.java | 6 ---- .../iotdb/lsm/strategy/BFSAccessStrategy.java | 1 - .../lsm/strategy/PostOrderAccessStrategy.java | 2 -- .../lsm/strategy/PreOrderAccessStrategy.java | 2 -- .../lsm/strategy/RBFSAccessStrategy.java | 10 +++---- schema-engine-tag/README.md | 1 + schema-engine-tag/pom.xml | 28 +++++++++++-------- .../tagSchemaRegion/MockTagSchemaRegion.java | 21 ++++---------- .../tagSchemaRegion/TagSchemaRegion.java | 21 ++++---------- .../AppendOnlyDeviceIDListFileManager.java | 1 - .../utils/MeasurementPathUtils.java | 1 - .../utils/PathTagConverterUtils.java | 1 - 15 files changed, 49 insertions(+), 85 deletions(-) diff --git a/lsm/pom.xml b/lsm/pom.xml index 081c60b0abc73..b95e98aea9d2b 100644 --- a/lsm/pom.xml +++ b/lsm/pom.xml @@ -1,19 +1,23 @@ diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java index 3f369b3017734..58aa3ed4e666e 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java @@ -23,25 +23,18 @@ public class Context { - // 类型 ContextType type; - // 访问策略 AccessStrategy accessStrategy; - // 所处的树深度 int level; - // 多少个线程处理该节点的子节点 int threadNums; - // 上界,大于该值的层级不会被处理 int levelUpperBound; - // 返回值 Object result; - // 是否正在recover boolean recover; public Context() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java index d0f50071747c1..f6438df45f4f8 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java @@ -21,16 +21,12 @@ import java.util.HashMap; import java.util.Map; -// 管理working memtable , immutable memtables,框架用户自定义 public class MemTableManager { - // 可写的memtable private MemTable working; - // 只读的memtables private Map immutables; - // 记录已插入的最大的deviceid private int maxDeviceID; public MemTableManager() { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java index 9ff3f0f0a03e3..c7d350754e9d4 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -21,14 +21,8 @@ import org.apache.iotdb.lsm.context.Context; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; -// 表示内存节点访问策略（先序，后序） public interface AccessStrategy { - /** - * @param levelProcess 保存当前节点和子节点的处理方法 - * @param memNode 当前待处理的节点 - * @param context 上下文信息 - */ void execute( BasicLevelProcess levelProcess, I memNode, C context); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java index cf794b7097e61..3b32027d61992 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java @@ -35,7 +35,6 @@ public void execute( BasicLevelProcess levelProcess, I memNode, C context) { List children = new ArrayList<>(); int currentLevel = context.getLevel(); - // 第一个使用bfs策略的节点 if (sameLevelMemNodes == null) { sameLevelMemNodes = new LinkedList<>(); levelProcess.handle(memNode, context); diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java index 3079041763701..6f86f8eb999f9 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java @@ -31,7 +31,6 @@ public void execute( int currentLevel = context.getLevel(); AccessStrategy accessStrategy = context.getAccessStrategy(); List children = levelProcess.getChildren(memNode, context); - // 处理子节点 if (levelProcess.hasNext()) { context.setLevel(currentLevel + 1); for (O child : children) { @@ -41,7 +40,6 @@ public void execute( context.setLevel(currentLevel); context.setAccessStrategy(accessStrategy); - // 处理该节点 levelProcess.handle(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java index 2c0b1b7b03a6c..41b7861b6e2fa 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -29,11 +29,9 @@ public class PreOrderAccessStrategy implements AccessStrategy { public void execute( BasicLevelProcess levelProcess, I memNode, C context) { int currentLevel = context.getLevel(); - // 处理该节点 levelProcess.handle(memNode, context); List children = levelProcess.getChildren(memNode, context); - // 处理子节点 if (levelProcess.hasNext()) { context.setLevel(currentLevel + 1); for (O child : children) { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index 62d62295786fc..801a733eff2a0 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -31,28 +31,26 @@ public void execute( if (Integer.MAX_VALUE == context.getLevelUpperBound() && !levelProcess.hasNext()) { context.setLevelUpperBound(context.getLevel()); } - // 如果是根节点 + if (currentLevel == 0) { while (context.getLevelUpperBound() != currentLevel) { List children = levelProcess.getChildren(memNode, context); for (O child : children) { - // 处理子节点 + context.setLevel(currentLevel + 1); levelProcess.getNext().process(child, context); context.setLevel(currentLevel); } - // 每次处理完-1 + context.setLevelUpperBound(context.getLevelUpperBound() - 1); } - // 处理root节点 + levelProcess.handle(memNode, context); return; } - // 已经处理过，直接return if (currentLevel > context.getLevelUpperBound()) return; - // 处理子节点 if (currentLevel == context.getLevelUpperBound()) { levelProcess.handle(memNode, context); return; diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md index 70185b9c79731..713e899d49dc8 100644 --- a/schema-engine-tag/README.md +++ b/schema-engine-tag/README.md @@ -18,6 +18,7 @@ under the License. --> + `TagSchemaRegion` is an implementation of `SchemaRegion`. # How To Use diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml index 14e61f754aa21..184921f098956 100644 --- a/schema-engine-tag/pom.xml +++ b/schema-engine-tag/pom.xml @@ -1,19 +1,23 @@ diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java index bb1fcce53987b..e78091b9460f9 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion; import org.apache.iotdb.common.rpc.thrift.TSchemaNode; @@ -226,7 +225,7 @@ private void createAlignedTimeSeries( prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); } - @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + @Override public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { PartialPath devicePath = plan.getPath().getDevicePath(); Map tags = pathToTags(devicePath.getFullPath()); @@ -250,7 +249,7 @@ public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws Meta } } - @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + @Override public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { PartialPath devicePath = plan.getPrefixPath(); Map tags = pathToTags(devicePath.getFullPath()); @@ -414,7 +413,7 @@ public Set getBelongedDevices(PartialPath timeseries) throws Metada throw new UnsupportedOperationException(""); } - @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] + @Override public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); @@ -459,15 +458,10 @@ public Pair, Integer> getMatchedDevices(ShowDevicesPlan throw new UnsupportedOperationException(""); } - @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select - // .. groupby level] + @Override public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { PartialPath devicePath = pathPattern.getDevicePath(); - // 批量查询.路径以".**"结尾,如： - // root.sg.tag1.a.** - // root.sg.tagx.c.tag2.v.** - // 点查询.路径不以".**",直接走IDTable,精确查询 if (devicePath.getFullPath().endsWith(TAIL)) { return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); } else { @@ -547,7 +541,6 @@ private List getMeasurementPathsWithBatchQuery( return res; } - // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] @Override public Pair, Integer> getMeasurementPathsWithAlias( PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) @@ -563,8 +556,7 @@ public List fetchSchema( return null; } - // show 时间序列 - @Override // [iotdb/show timeseries] + @Override public Pair, Integer> showTimeseries( ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { List res = new ArrayList<>(); @@ -745,8 +737,7 @@ public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fu throw new UnsupportedOperationException(""); } - // insert data - @Override // [iotdb/insert ] + @Override public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) throws MetadataException, IOException { PartialPath devicePath = plan.getDevicePath(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index 7ad604a9c218c..06eed3e5eb51c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion; import org.apache.iotdb.common.rpc.thrift.TSchemaNode; @@ -238,7 +237,7 @@ private void createAlignedTimeSeries( prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); } - @Override // [iotdb|newIotdb/创建非对齐时间序列] [newIotdb/insert 2自动创建时间序列] + @Override public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { PartialPath devicePath = plan.getPath().getDevicePath(); PartialPath path = @@ -266,7 +265,7 @@ public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws Meta } } - @Override // [iotdb|newIotdb/对齐时间序列] [newIotdb/insert 2自动创建时间序列] + @Override public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { PartialPath devicePath = plan.getPrefixPath(); PartialPath path = @@ -444,7 +443,7 @@ public Set getBelongedDevices(PartialPath timeseries) throws Metada throw new UnsupportedOperationException("getBelongedDevices"); } - @Override // [newIotdb/show timeseries] [newIotdb/count device] [newIotdb/count timeseries] + @Override public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); @@ -534,15 +533,10 @@ public Pair, Integer> getMatchedDevices(ShowDevicesPlan throw new UnsupportedOperationException("getMatchedDevices"); } - @Override // [newIotDB / insert1,3] [newIotDB/select] [newIotdb/select count()] [newIotdb/select - // .. groupby level] + @Override public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) throws MetadataException { PartialPath devicePath = pathPattern.getDevicePath(); - // 批量查询.路径以".**"结尾,如： - // root.sg.tag1.a.** - // root.sg.tagx.c.tag2.v.** - // 点查询.路径不以".**",直接走IDTable,精确查询 if (devicePath.getFullPath().endsWith(TAIL)) { return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); } else { @@ -573,7 +567,6 @@ private List getMeasurementPathsWithBatchQuery( return getMeasurementPaths(deviceIDs); } - // [iotdb/select] [iotdb/select last] [iotdb/select count()] [iotdb/select ...groupby level] @Override public Pair, Integer> getMeasurementPathsWithAlias( PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) @@ -589,8 +582,7 @@ public List fetchSchema( throw new UnsupportedOperationException("fetchSchema"); } - // show 时间序列 - @Override // [iotdb/show timeseries] + @Override public Pair, Integer> showTimeseries( ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { List ShowTimeSeriesResults = new ArrayList<>(); @@ -730,8 +722,7 @@ public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fu throw new UnsupportedOperationException("renameTagOrAttributeKey"); } - // insert data - @Override // [iotdb/insert ] + @Override public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) throws MetadataException, IOException { PartialPath devicePath = plan.getDevicePath(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java index 32475d650c560..cdbcaea16ed68 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; import org.apache.iotdb.commons.utils.TestOnly; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java index 22ae4c3b58162..5c792000d0b9b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; import org.apache.iotdb.commons.exception.IllegalPathException; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java index bd2c3a4e9ab58..59fbc6409ed6f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; import java.util.Map; From db42af5aca6426ab1d1c430a62471ad6248cc7fe Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 13:44:47 +0800 Subject: [PATCH 26/53] mod Licensed --- lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java | 3 ++- .../deviceidlist/AppendOnlyDeviceIDListFileManager.java | 1 + .../tagSchemaRegion/deviceidlist/DeviceIDListTest.java | 6 +++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java index 42932301d0dc4..858c2c219dd7d 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -33,7 +33,7 @@ public class WALReader implements IWALReader { private static final Logger logger = LoggerFactory.getLogger(WALReader.class); private final File logFile; private final WALRecord prototype; - private final DataInputStream logStream; + private DataInputStream logStream; private WALRecord nextRecord; private boolean fileCorrupted = false; @@ -47,6 +47,7 @@ public WALReader(File logFile, WALRecord prototype) throws IOException { @Override public void close() throws IOException { logStream.close(); + logStream = null; } @Override diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java index cdbcaea16ed68..9753cf013dddb 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java @@ -96,6 +96,7 @@ public void recover(DeviceIDList deviceIDList) { public void close() throws IOException { try { outputStream.close(); + outputStream = null; } catch (IOException e) { logger.error("close device id list file failed"); throw e; diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java index eddc31d2f9e08..26daa70890f1d 100644 --- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java @@ -29,6 +29,7 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -100,12 +101,15 @@ public void testAddandGetDeviceID() { } @Test - public void testRecover() { + public void testRecover() throws IOException { List deviceIDS = generateTestDeviceIDS(); for (IDeviceID deviceID : deviceIDS) { deviceIDList.add(deviceID); System.out.println(deviceID.toStringID()); } + + deviceIDList.clear(); + deviceIDList = new DeviceIDList(schemaRegionDirPath); for (int i = 0; i < deviceIDS.size(); i++) { assertEquals(deviceIDS.get(i), deviceIDList.get(i)); From 4283b2696397584e2f917384650b40dbbd183b40 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 21:23:34 +0800 Subject: [PATCH 27/53] add README.md --- lsm/README.md | 396 ++++++++++++++++++ ...Context.java => DeleteRequestContext.java} | 18 +- ...hContext.java => FlushRequestContext.java} | 11 +- ...Context.java => InsertRequestContext.java} | 18 +- ...yContext.java => QueryRequestContext.java} | 12 +- .../{Context.java => RequestContext.java} | 31 +- .../{ContextType.java => RequestType.java} | 2 +- .../org/apache/iotdb/lsm/example/Main.java | 209 --------- .../apache/iotdb/lsm/example/MemChunk.java | 44 -- .../apache/iotdb/lsm/example/MemGroup.java | 45 -- .../apache/iotdb/lsm/example/MemTable.java | 45 -- .../iotdb/lsm/example/MemTableManager.java | 71 ---- .../lsm/levelProcess/BasicLevelProcess.java | 33 +- .../lsm/levelProcess/DeleteLevelProcess.java | 16 +- .../lsm/levelProcess/FlushLevelProcess.java | 15 +- .../lsm/levelProcess/InsertLevelProcess.java | 16 +- .../iotdb/lsm/levelProcess/LevelProcess.java | 20 +- .../lsm/levelProcess/QueryLevelProcess.java | 15 +- .../iotdb/lsm/manager/BasicLsmManager.java | 33 +- .../apache/iotdb/lsm/manager/LsmManager.java | 17 +- .../iotdb/lsm/strategy/AccessStrategy.java | 12 +- .../iotdb/lsm/strategy/BFSAccessStrategy.java | 15 +- .../lsm/strategy/PostOrderAccessStrategy.java | 15 +- .../lsm/strategy/PreOrderAccessStrategy.java | 15 +- .../lsm/strategy/RBFSAccessStrategy.java | 26 +- .../org/apache/iotdb/lsm/wal/IWALReader.java | 17 + .../org/apache/iotdb/lsm/wal/IWALWriter.java | 17 + .../org/apache/iotdb/lsm/wal/WALReader.java | 6 + .../org/apache/iotdb/lsm/wal/WALRecord.java | 12 + .../org/apache/iotdb/lsm/wal/WALWriter.java | 10 + schema-engine-tag/README.md | 2 +- .../tagSchemaRegion/MockTagSchemaRegion.java | 1 + .../tagSchemaRegion/TagSchemaConfig.java | 3 + .../tagSchemaRegion/TagSchemaDescriptor.java | 12 +- .../tagSchemaRegion/TagSchemaRegion.java | 10 +- .../AppendOnlyDeviceIDListFileManager.java | 17 +- .../deviceidlist/DeviceIDList.java | 4 +- .../deviceidlist/IDeviceIDList.java | 22 + .../tagIndex/ITagInvertedIndex.java | 37 +- .../tagIndex/TagInvertedIndex.java | 61 ++- .../tagIndex/deletion/DeletionManager.java | 16 +- .../tagIndex/deletion/MemChunkDeletion.java | 21 +- .../deletion/MemChunkGroupDeletion.java | 21 +- .../tagIndex/deletion/MemTableDeletion.java | 21 +- .../tagIndex/insertion/InsertionManager.java | 16 +- .../insertion/MemChunkGroupInsertion.java | 21 +- .../tagIndex/insertion/MemChunkInsertion.java | 21 +- .../tagIndex/insertion/MemTableInsertion.java | 21 +- .../tagIndex/memtable/MemChunk.java | 4 +- .../tagIndex/memtable/MemChunkGroup.java | 3 + .../tagIndex/memtable/MemTable.java | 5 + .../tagIndex/query/MemChunkGroupQuery.java | 21 +- .../tagIndex/query/MemChunkQuery.java | 21 +- .../tagIndex/query/MemTableQuery.java | 22 +- .../tagIndex/query/QueryManager.java | 6 +- .../tagIndex/recover/RecoverManager.java | 18 +- .../tagIndex/wal/WALEntry.java | 16 + .../tagIndex/wal/WALManager.java | 71 +++- .../utils/MeasurementPathUtils.java | 19 + .../utils/PathTagConverterUtils.java | 22 + .../utils/ShowTimeSeriesResultUtils.java | 19 + 61 files changed, 1219 insertions(+), 567 deletions(-) create mode 100644 lsm/README.md rename lsm/src/main/java/org/apache/iotdb/lsm/context/{DeleteContext.java => DeleteRequestContext.java} (76%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{FlushContext.java => FlushRequestContext.java} (74%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{InsertContext.java => InsertRequestContext.java} (76%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{QueryContext.java => QueryRequestContext.java} (78%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{Context.java => RequestContext.java} (77%) rename lsm/src/main/java/org/apache/iotdb/lsm/context/{ContextType.java => RequestType.java} (97%) delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java delete mode 100644 lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java diff --git a/lsm/README.md b/lsm/README.md new file mode 100644 index 0000000000000..d1c2ae75975fb --- /dev/null +++ b/lsm/README.md @@ -0,0 +1,396 @@ + + +# LSM Document +
+  _____      ______   ____    ____  
+ |_   _|   .' ____ \ |_   \  /   _| 
+   | |     | (___ \_|  |   \/   |   
+   | |   _  _.____`.   | |\  /| |   
+  _| |__/ || \____) | _| |_\/_| |_  
+ |________| \______.'|_____||_____| > version 0.14.0-SNAPSHOT
+                                    
+
+## Abstract + +The lsm framework has implemented the **memory structure** at present, and users only need to define the memory structure and access method of each level. + + +## Example +Suppose we need to implement an lsm storage engine to store each record similar to , we define the following four layers of memory nodes. +### Memory structure +Implement the memory structure of each layer + +- MemChunk + +The last layer of memory nodes, save the id list +```java +public class MemChunk { + List deviceIDS; + + public MemChunk() { + deviceIDS = new ArrayList<>(); + } + + public List getDeviceIDS() { + return deviceIDS; + } + + public void setDeviceIDS(List deviceIDS) { + this.deviceIDS = deviceIDS; + } + + @Override + public String toString() { + return "MemChunk{" + deviceIDS.toString() + '}'; + } +} +``` + +- MemGroup + +Use a map to manage tagValue->MemChunk +```java +public class MemGroup { + + Map map; + + public MemGroup() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemGroup{" + map.toString() + '}'; + } +} +``` + +- MemTable + +Use a map to manage tagKey->MemGroup +```java +public class MemTable { + + private Map map; + + public MemTable() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemTable{" + map.toString() + '}'; + } +} +``` + +- MemTableManager + +Manage working memTable, and immutable memTable +```java +public class MemTableManager { + + private MemTable working; + + private Map immutables; + + private int maxDeviceID; + + public MemTableManager() { + working = new MemTable(); + immutables = new HashMap<>(); + maxDeviceID = 0; + } + + public MemTable getWorking() { + return working; + } + + public void setWorking(MemTable working) { + this.working = working; + } + + public Map getImmutables() { + return immutables; + } + + public void setImmutables(Map immutables) { + this.immutables = immutables; + } + + public int getMaxDeviceID() { + return maxDeviceID; + } + + public void setMaxDeviceID(int maxDeviceID) { + this.maxDeviceID = maxDeviceID; + } + + @Override + public String toString() { + return "MemTableManager{" + + "working=" + + working.toString() + + ", immutables=" + + immutables.toString() + + '}'; + } +} +``` + +### Access method +Incoming access to each layer for the framework + +- Insertion and flush example +```java +public class Main { + public static void main(String[] args) throws Exception { + MemTableManager memTableManager = new MemTableManager(); + System.out.println("-------------insert--------------"); + insertionExample(memTableManager); + System.out.println("-------------flush--------------"); + flushExample(memTableManager); + } + + public static void insertionExample(MemTableManager memTableManager) throws Exception { + // Initialize a BasicLsmManager to manage insert operations + BasicLsmManager baseLsmManager = + new BasicLsmManager(); + baseLsmManager + .nextLevel( + // The insert method of the MemTableManager level + new InsertLevelProcess() { + @Override + public List getChildren( + MemTableManager memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + List children = new ArrayList<>(); + if (deviceID / 65536 == maxDeviceID / 65536) { + children.add(memNode.getWorking()); + } else { + children.add(memNode.getImmutables().get(deviceID / 65536)); + } + return children; + } + + @Override + public void insert(MemTableManager memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + if (deviceID / 65536 == maxDeviceID / 65536) { + if (memNode.getWorking() == null) { + memNode.setWorking(new MemTable()); + } + } else if (deviceID > maxDeviceID) { + memNode + .getImmutables() + .put(memNode.getMaxDeviceID() / 65536, memNode.getWorking()); + memNode.setWorking(new MemTable()); + } + if (deviceID > maxDeviceID) { + memNode.setMaxDeviceID(deviceID); + } + } + }) + .nextLevel( + // The insert method of the MemTable level + new InsertLevelProcess() { + @Override + public List getChildren(MemTable memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemTable memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemGroup()); + } + }) + .nextLevel( + // The insert method of the MemGroup level + new InsertLevelProcess() { + @Override + public List getChildren(MemGroup memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemGroup memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemChunk()); + } + }) + .nextLevel( + // The insert method of the MemChunk level + new InsertLevelProcess() { + @Override + public List getChildren(MemChunk memNode, InsertRequestContext context) { + return null; + } + + @Override + public void insert(MemChunk memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + List deviceIDs = memNode.getDeviceIDS(); + deviceIDs.add(deviceID); + } + }); + + // Insert some records + // The key at the MemTableManager level is null + baseLsmManager.process(memTableManager, new InsertRequestContext(1, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertRequestContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertRequestContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(6, null, "a2", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(65535, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(65536, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertRequestContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertRequestContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(6, null, "a2", "b")); + // process memTableManager + System.out.println(memTableManager); + } + + public static void flushExample(MemTableManager memTableManager) throws Exception { + // Initialize a BasicLsmManager to manage insert operations + BasicLsmManager flushManager = + new BasicLsmManager(); + + flushManager + .nextLevel( + // The insert method of the MemTableManager level + new FlushLevelProcess() { + @Override + public void flush(MemTableManager memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren( + MemTableManager memNode, FlushRequestContext context) { + List memTables = new ArrayList<>(); + memTables.addAll(memNode.getImmutables().values()); + if (memNode.getWorking() != null) memTables.add(memNode.getWorking()); + return memTables; + } + }) + .nextLevel( + // The insert method of the MemTable level + new FlushLevelProcess() { + @Override + public void flush(MemTable memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren(MemTable memNode, FlushRequestContext context) { + List memGroups = new ArrayList<>(); + memGroups.addAll(memNode.getMap().values()); + return memGroups; + } + }) + .nextLevel( + // The insert method of the MemGroup level + new FlushLevelProcess() { + @Override + public void flush(MemGroup memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren(MemGroup memNode, FlushRequestContext context) { + List memChunk = new ArrayList<>(); + memChunk.addAll(memNode.getMap().values()); + return memChunk; + } + }) + .nextLevel( + // The insert method of the MemChunk level + new FlushLevelProcess() { + @Override + public void flush(MemChunk memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren(MemChunk memNode, FlushRequestContext context) { + return new ArrayList<>(); + } + }); + + // process memTableManager + flushManager.process(memTableManager, new FlushRequestContext()); + } +} + +``` +- Output + +```txt +-------------insert-------------- +MemTableManager{working=MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}, immutables={0=MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}}} +-------------flush-------------- +FLUSH: MemChunk{[5, 5]}-->[level:3] +FLUSH: MemChunk{[1, 4, 65535, 4]}-->[level:3] +FLUSH: MemChunk{[2, 2]}-->[level:3] +FLUSH: MemChunk{[3, 3]}-->[level:3] +FLUSH: MemChunk{[6, 6]}-->[level:3] +FLUSH: MemChunk{[65536]}-->[level:3] +FLUSH: MemGroup{{b=MemChunk{[5, 5]}}}-->[level:2] +FLUSH: MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}-->[level:2] +FLUSH: MemGroup{{b=MemChunk{[6, 6]}}}-->[level:2] +FLUSH: MemGroup{{b=MemChunk{[65536]}}}-->[level:2] +FLUSH: MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}-->[level:1] +FLUSH: MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}-->[level:1] +FLUSH: MemTableManager{working=MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}, immutables={0=MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}}}-->[level:0] +``` diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java similarity index 76% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java index 9b4b30ac41a75..548874625265d 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java @@ -24,24 +24,32 @@ import java.util.Arrays; import java.util.List; -public class DeleteContext extends Context { +/** + * represents the context of a deletion request, this class can be extended to implement a custom + * context + */ +public class DeleteRequestContext extends RequestContext { + // save the key of each level List keys; + // value to delete Object value; - public DeleteContext() { + public DeleteRequestContext() { super(); - type = ContextType.DELETE; + type = RequestType.DELETE; + // post-order traversal strategy is used by default accessStrategy = new PostOrderAccessStrategy(); } - public DeleteContext(Object value, Object... ks) { + public DeleteRequestContext(Object value, Object... ks) { super(); this.value = value; keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); - type = ContextType.DELETE; + type = RequestType.DELETE; + // post-order traversal strategy is used by default accessStrategy = new PostOrderAccessStrategy(); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java similarity index 74% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java index 2fd266396c8f8..d2a9cd8bae10d 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java @@ -20,8 +20,15 @@ import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; -public class FlushContext extends Context { - public FlushContext() { +/** + * represents the context of a flush request, this class can be extended to implement a custom + * context + */ +public class FlushRequestContext extends RequestContext { + public FlushRequestContext() { + super(); + type = RequestType.FLUSH; + // use the reverse breadth-first traversal strategy to access memory nodes accessStrategy = new RBFSAccessStrategy(); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java similarity index 76% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java index b09d1540cbc47..933f006a58b0e 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java @@ -24,24 +24,32 @@ import java.util.Arrays; import java.util.List; -public class InsertContext extends Context { +/** + * represents the context of a insertion request, this class can be extended to implement a custom + * context + */ +public class InsertRequestContext extends RequestContext { + // save the key of each level List keys; + // value to insert Object value; - public InsertContext() { + public InsertRequestContext() { super(); - type = ContextType.INSERT; + type = RequestType.INSERT; + // preorder traversal strategy is used by default accessStrategy = new PreOrderAccessStrategy(); } - public InsertContext(Object value, Object... keys) { + public InsertRequestContext(Object value, Object... keys) { super(); this.value = value; this.keys = new ArrayList<>(); this.keys.addAll(Arrays.asList(keys)); - type = ContextType.INSERT; + type = RequestType.INSERT; + // preorder traversal strategy is used by default accessStrategy = new PreOrderAccessStrategy(); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java similarity index 78% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java index 7221b643f7fac..8742ac4e2ace8 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryContext.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java @@ -24,15 +24,21 @@ import java.util.Arrays; import java.util.List; -public class QueryContext extends Context { +/** + * represents the context of a query request, this class can be extended to implement a custom + * context + */ +public class QueryRequestContext extends RequestContext { + // save the key of each level List keys; - public QueryContext(Object... ks) { + public QueryRequestContext(Object... ks) { super(); keys = new ArrayList<>(); keys.addAll(Arrays.asList(ks)); - type = ContextType.QUERY; + type = RequestType.QUERY; + // post-order traversal strategy is used by default accessStrategy = new PostOrderAccessStrategy(); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java similarity index 77% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java index 58aa3ed4e666e..4866be0ca1fd9 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/Context.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java @@ -21,27 +21,32 @@ import org.apache.iotdb.lsm.strategy.AccessStrategy; import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; -public class Context { +/** represents the context of a request */ +public class RequestContext { - ContextType type; + // request type + RequestType type; + // memory Structure Access Policy AccessStrategy accessStrategy; + // the tree level of the currently pending memory node int level; - int threadNums; - + // the maximum level of memory nodes that can be processed int levelUpperBound; + // return value after request processing is complete Object result; + // whether the request context is only used for recovery boolean recover; - public Context() { + public RequestContext() { + // preorder traversal strategy is used by default accessStrategy = new PreOrderAccessStrategy(); - type = ContextType.NONE; + type = RequestType.NONE; level = 0; - threadNums = 1; levelUpperBound = Integer.MAX_VALUE; recover = false; } @@ -54,22 +59,14 @@ public int getLevel() { return level; } - public ContextType getType() { + public RequestType getType() { return type; } - public int getThreadNums() { - return threadNums; - } - - public void setType(ContextType type) { + public void setType(RequestType type) { this.type = type; } - public void setThreadNums(int threadNums) { - this.threadNums = threadNums; - } - public Object getResult() { return result; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java similarity index 97% rename from lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java rename to lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java index 3ff2e3509afb1..e7dfac6c2a9e9 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/context/ContextType.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.lsm.context; -public enum ContextType { +public enum RequestType { NONE, INSERT, QUERY, diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java deleted file mode 100644 index 7dbe793ac5fcd..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/Main.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import org.apache.iotdb.lsm.context.FlushContext; -import org.apache.iotdb.lsm.context.InsertContext; -import org.apache.iotdb.lsm.levelProcess.FlushLevelProcess; -import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; -import org.apache.iotdb.lsm.manager.BasicLsmManager; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class Main { - public static void main(String[] args) throws Exception { - MemTableManager memTableManager = new MemTableManager(); - System.out.println("-------------insert--------------"); - insertionExample(memTableManager); - System.out.println("-------------flush--------------"); - flushExample(memTableManager); - } - - public static void insertionExample(MemTableManager memTableManager) throws Exception { - - BasicLsmManager baseLsmManager = - new BasicLsmManager(); - baseLsmManager - .nextLevel( - new InsertLevelProcess() { - @Override - public List getChildren(MemTableManager memNode, InsertContext context) { - Integer deviceID = (Integer) context.getValue(); - int maxDeviceID = memNode.getMaxDeviceID(); - List children = new ArrayList<>(); - if (deviceID / 65536 == maxDeviceID / 65536) { - children.add(memNode.getWorking()); - } else { - children.add(memNode.getImmutables().get(deviceID / 65536)); - } - return children; - } - - @Override - public void insert(MemTableManager memNode, InsertContext context) { - Integer deviceID = (Integer) context.getValue(); - int maxDeviceID = memNode.getMaxDeviceID(); - if (deviceID / 65536 == maxDeviceID / 65536) { - if (memNode.getWorking() == null) { - memNode.setWorking(new MemTable()); - } - } else if (deviceID > maxDeviceID) { - memNode - .getImmutables() - .put(memNode.getMaxDeviceID() / 65536, memNode.getWorking()); - memNode.setWorking(new MemTable()); - } - if (deviceID > maxDeviceID) { - memNode.setMaxDeviceID(deviceID); - } - } - }) - .nextLevel( - new InsertLevelProcess() { - @Override - public List getChildren(MemTable memNode, InsertContext context) { - String key = (String) context.getKey(); - List children = new ArrayList<>(); - children.add(memNode.getMap().get(key)); - return children; - } - - @Override - public void insert(MemTable memNode, InsertContext context) { - String key = (String) context.getKey(); - Map map = memNode.getMap(); - if (map.containsKey(key)) return; - map.put(key, new MemGroup()); - } - }) - .nextLevel( - new InsertLevelProcess() { - @Override - public List getChildren(MemGroup memNode, InsertContext context) { - String key = (String) context.getKey(); - List children = new ArrayList<>(); - children.add(memNode.getMap().get(key)); - return children; - } - - @Override - public void insert(MemGroup memNode, InsertContext context) { - String key = (String) context.getKey(); - Map map = memNode.getMap(); - if (map.containsKey(key)) return; - map.put(key, new MemChunk()); - } - }) - .nextLevel( - new InsertLevelProcess() { - @Override - public List getChildren(MemChunk memNode, InsertContext context) { - return null; - } - - @Override - public void insert(MemChunk memNode, InsertContext context) { - Integer deviceID = (Integer) context.getValue(); - List deviceIDs = memNode.getDeviceIDS(); - deviceIDs.add(deviceID); - } - }); - - baseLsmManager.process(memTableManager, new InsertContext(1, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); - baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); - baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); - baseLsmManager.process(memTableManager, new InsertContext(65535, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(65536, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(2, null, "a", "d")); - baseLsmManager.process(memTableManager, new InsertContext(3, null, "a", "e")); - baseLsmManager.process(memTableManager, new InsertContext(4, null, "a", "b")); - baseLsmManager.process(memTableManager, new InsertContext(5, null, "a1", "b")); - baseLsmManager.process(memTableManager, new InsertContext(6, null, "a2", "b")); - System.out.println(memTableManager); - } - - public static void flushExample(MemTableManager memTableManager) throws Exception { - BasicLsmManager flushManager = - new BasicLsmManager(); - - flushManager - .nextLevel( - new FlushLevelProcess() { - @Override - public void flush(MemTableManager memNode, FlushContext context) { - System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); - } - - @Override - public List getChildren(MemTableManager memNode, FlushContext context) { - List memTables = new ArrayList<>(); - memTables.addAll(memNode.getImmutables().values()); - if (memNode.getWorking() != null) memTables.add(memNode.getWorking()); - return memTables; - } - }) - .nextLevel( - new FlushLevelProcess() { - @Override - public void flush(MemTable memNode, FlushContext context) { - System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); - } - - @Override - public List getChildren(MemTable memNode, FlushContext context) { - List memGroups = new ArrayList<>(); - memGroups.addAll(memNode.getMap().values()); - return memGroups; - } - }) - .nextLevel( - new FlushLevelProcess() { - @Override - public void flush(MemGroup memNode, FlushContext context) { - System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); - } - - @Override - public List getChildren(MemGroup memNode, FlushContext context) { - List memChunk = new ArrayList<>(); - memChunk.addAll(memNode.getMap().values()); - return memChunk; - } - }) - .nextLevel( - new FlushLevelProcess() { - @Override - public void flush(MemChunk memNode, FlushContext context) { - System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); - } - - @Override - public List getChildren(MemChunk memNode, FlushContext context) { - return new ArrayList<>(); - } - }); - - flushManager.process(memTableManager, new FlushContext()); - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java deleted file mode 100644 index c6e97fd52d9d2..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemChunk.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import java.util.ArrayList; -import java.util.List; - -// device list -public class MemChunk { - List deviceIDS; - - public MemChunk() { - deviceIDS = new ArrayList<>(); - } - - public List getDeviceIDS() { - return deviceIDS; - } - - public void setDeviceIDS(List deviceIDS) { - this.deviceIDS = deviceIDS; - } - - @Override - public String toString() { - return "MemChunk{" + deviceIDS.toString() + '}'; - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java deleted file mode 100644 index 69f61d5d451fa..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemGroup.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import java.util.HashMap; -import java.util.Map; - -// tagvalue -> memChunk -public class MemGroup { - - Map map; - - public MemGroup() { - map = new HashMap<>(); - } - - public Map getMap() { - return map; - } - - public void setMap(Map map) { - this.map = map; - } - - @Override - public String toString() { - return "MemGroup{" + map.toString() + '}'; - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java deleted file mode 100644 index 7cc82f19ecab7..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTable.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import java.util.HashMap; -import java.util.Map; - -// tagkey - > MemGroup -public class MemTable { - - private Map map; - - public MemTable() { - map = new HashMap<>(); - } - - public Map getMap() { - return map; - } - - public void setMap(Map map) { - this.map = map; - } - - @Override - public String toString() { - return "MemTable{" + map.toString() + '}'; - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java deleted file mode 100644 index f6438df45f4f8..0000000000000 --- a/lsm/src/main/java/org/apache/iotdb/lsm/example/MemTableManager.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.lsm.example; - -import java.util.HashMap; -import java.util.Map; - -public class MemTableManager { - - private MemTable working; - - private Map immutables; - - private int maxDeviceID; - - public MemTableManager() { - working = new MemTable(); - immutables = new HashMap<>(); - maxDeviceID = 0; - } - - public MemTable getWorking() { - return working; - } - - public void setWorking(MemTable working) { - this.working = working; - } - - public Map getImmutables() { - return immutables; - } - - public void setImmutables(Map immutables) { - this.immutables = immutables; - } - - public int getMaxDeviceID() { - return maxDeviceID; - } - - public void setMaxDeviceID(int maxDeviceID) { - this.maxDeviceID = maxDeviceID; - } - - @Override - public String toString() { - return "MemTableManager{" - + "working=" - + working.toString() - + ", immutables=" - + immutables.toString() - + '}'; - } -} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java index f4ced68ec5a08..4e10ba794bdc3 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java @@ -18,23 +18,52 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import java.util.List; -public abstract class BasicLevelProcess implements LevelProcess { +/** the processing method corresponding to each layer of memory nodes */ +public abstract class BasicLevelProcess + implements LevelProcess { + + // the next level process LevelProcess next; + /** + * process the current layer memory node + * + * @param memNode memory node + * @param context request context + */ public abstract void handle(I memNode, C context); + /** + * get the memory node that needs to be processed in the next layer + * + * @param memNode memory node + * @param context request context + * @return all next-level memory nodes that need to be processed + */ public abstract List getChildren(I memNode, C context); + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return the next level process + */ @Override public LevelProcess nextLevel(LevelProcess next) { this.next = next; return next; } + /** + * use this method to process memory nodes at each layer according to the access strategy + * + * @param memNode memory node + * @param context request context + */ @Override public void process(I memNode, C context) { context.getAccessStrategy().execute(this, memNode, context); diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java index f4a4c97cd0f5c..bd71d7fc86792 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java @@ -18,14 +18,22 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; -public abstract class DeleteLevelProcess extends BasicLevelProcess { +/** indicates the deletion method of each layer of memory nodes */ +public abstract class DeleteLevelProcess + extends BasicLevelProcess { - public abstract void delete(I memNode, DeleteContext context); + /** + * the deletion method of memory node + * + * @param memNode memory node + * @param context deletion request context + */ + public abstract void delete(I memNode, DeleteRequestContext context); @Override - public void handle(I memNode, DeleteContext context) { + public void handle(I memNode, DeleteRequestContext context) { delete(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java index 00363c7683b75..46d72b4b5b50e 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java @@ -18,13 +18,20 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.FlushContext; +import org.apache.iotdb.lsm.context.FlushRequestContext; -public abstract class FlushLevelProcess extends BasicLevelProcess { +/** indicates the flush method of each layer of memory nodes */ +public abstract class FlushLevelProcess extends BasicLevelProcess { - public abstract void flush(I memNode, FlushContext context); + /** + * the flush method of memory node + * + * @param memNode memory node + * @param context flush request context + */ + public abstract void flush(I memNode, FlushRequestContext context); - public void handle(I memNode, FlushContext context) { + public void handle(I memNode, FlushRequestContext context) { flush(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java index 91f7e24c1f250..cf7d9c69bb772 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java @@ -18,14 +18,22 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; -public abstract class InsertLevelProcess extends BasicLevelProcess { +/** indicates the insertion method of each layer of memory nodes */ +public abstract class InsertLevelProcess + extends BasicLevelProcess { - public abstract void insert(I memNode, InsertContext context); + /** + * the insertion method of memory node + * + * @param memNode memory node + * @param context insertion request context + */ + public abstract void insert(I memNode, InsertRequestContext context); @Override - public void handle(I memNode, InsertContext context) { + public void handle(I memNode, InsertRequestContext context) { insert(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java index aa04c2bf172b6..763d6eb4f6d32 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java @@ -18,12 +18,24 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; -public interface LevelProcess { +/** the processing method corresponding to each layer of memory nodes */ +public interface LevelProcess { + + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return LevelProcess of the next layer + */ LevelProcess nextLevel(LevelProcess next); + /** + * use this method to process memory nodes at each layer according to the access strategy + * + * @param memNode memory node + * @param context request context + */ void process(I memNode, C context); - - void handle(I memNode, C context); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java index 5b3fc0e9ebb32..2fef2f213d058 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java @@ -18,14 +18,21 @@ */ package org.apache.iotdb.lsm.levelProcess; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; -public abstract class QueryLevelProcess extends BasicLevelProcess { +/** indicates the query method of each layer of memory nodes */ +public abstract class QueryLevelProcess extends BasicLevelProcess { - public abstract void query(I memNode, QueryContext context); + /** + * the query method of memory node + * + * @param memNode memory node + * @param context query request context + */ + public abstract void query(I memNode, QueryRequestContext context); @Override - public void handle(I memNode, QueryContext context) { + public void handle(I memNode, QueryRequestContext context) { query(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java index 1855f04c0fe48..5b8816f7f2f26 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -18,17 +18,40 @@ */ package org.apache.iotdb.lsm.manager; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -public class BasicLsmManager implements LsmManager { +/** */ +public class BasicLsmManager implements LsmManager { + // the level process of the first layer of memory nodes LevelProcess levelProcess; + /** + * preprocessing of the root memory node + * + * @param root root memory node + * @param context request context + * @throws Exception + */ public void preProcess(T root, C context) throws Exception {} + /** + * postprocessing of the root memory node + * + * @param root root memory node + * @param context request context + * @throws Exception + */ public void postProcess(T root, C context) throws Exception {} + /** + * processing of the root memory node + * + * @param root root memory node + * @param context request context + * @throws Exception + */ @Override public void process(T root, C context) throws Exception { preProcess(root, context); @@ -36,6 +59,12 @@ public void process(T root, C context) throws Exception { postProcess(root, context); } + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param levelProcess LevelProcess of the next layer + * @return LevelProcess of the next layer + */ @Override public LevelProcess nextLevel(LevelProcess levelProcess) { this.levelProcess = levelProcess; diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java index 2940ad60b1751..ea2355378e5d6 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java @@ -18,12 +18,25 @@ */ package org.apache.iotdb.lsm.manager; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.LevelProcess; -public interface LsmManager { +// used to implement lsm manager +public interface LsmManager { + /** + * use this method to process root memory node + * + * @param memNode memory node + * @param context request context + */ void process(T memNode, C context) throws Exception; + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return LevelProcess of the next layer + */ LevelProcess nextLevel(LevelProcess next); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java index c7d350754e9d4..567f83faa3e3b 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -18,11 +18,19 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; +/** access strategy for memory nodes */ public interface AccessStrategy { - void execute( + /** + * implementation of access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + void execute( BasicLevelProcess levelProcess, I memNode, C context); } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java index 3b32027d61992..e93bd5ca959e6 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.ArrayList; @@ -26,18 +26,29 @@ import java.util.List; import java.util.Queue; +/** breadth-first access strategy implementation class */ public class BFSAccessStrategy implements AccessStrategy { + // same level memory nodes, used to implement BFSAccessStrategy Queue sameLevelMemNodes; + /** + * breadth-first access strategy implementation + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ @Override - public void execute( + public void execute( BasicLevelProcess levelProcess, I memNode, C context) { List children = new ArrayList<>(); int currentLevel = context.getLevel(); if (sameLevelMemNodes == null) { sameLevelMemNodes = new LinkedList<>(); + // process the current memory node levelProcess.handle(memNode, context); + // get all memory nodes to be processed in the next layer children = levelProcess.getChildren(memNode, context); } else { while (!sameLevelMemNodes.isEmpty()) { diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java index 6f86f8eb999f9..5e96f29b1b451 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java @@ -18,28 +18,39 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.List; +/** post-order traversal access strategy implementation class */ public class PostOrderAccessStrategy implements AccessStrategy { + /** + * post-order traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ @Override - public void execute( + public void execute( BasicLevelProcess levelProcess, I memNode, C context) { int currentLevel = context.getLevel(); AccessStrategy accessStrategy = context.getAccessStrategy(); + // get all memory nodes to be processed in the next layer List children = levelProcess.getChildren(memNode, context); if (levelProcess.hasNext()) { context.setLevel(currentLevel + 1); for (O child : children) { + // process next level memory node levelProcess.getNext().process(child, context); } } context.setLevel(currentLevel); context.setAccessStrategy(accessStrategy); + // process the current memory node levelProcess.handle(memNode, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java index 41b7861b6e2fa..d16780cf59e58 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -18,23 +18,34 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.List; +/** pre-order traversal access strategy implementation class */ public class PreOrderAccessStrategy implements AccessStrategy { + /** + * pre-order traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ @Override - public void execute( + public void execute( BasicLevelProcess levelProcess, I memNode, C context) { int currentLevel = context.getLevel(); + // process the current memory node levelProcess.handle(memNode, context); + // get all memory nodes to be processed in the next layer List children = levelProcess.getChildren(memNode, context); if (levelProcess.hasNext()) { context.setLevel(currentLevel + 1); for (O child : children) { + // process next level memory node levelProcess.getNext().process(child, context); } } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java index 801a733eff2a0..13b05c51717ae 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -18,43 +18,63 @@ */ package org.apache.iotdb.lsm.strategy; -import org.apache.iotdb.lsm.context.Context; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; import java.util.List; +/** reverse breadth first traversal access strategy implementation class */ public class RBFSAccessStrategy implements AccessStrategy { + + /** + * reverse breadth first traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ @Override - public void execute( + public void execute( BasicLevelProcess levelProcess, I memNode, C context) { int currentLevel = context.getLevel(); + + // if the upper bound has not been set and there is no next-level processing method, set the + // upper bound to the current level if (Integer.MAX_VALUE == context.getLevelUpperBound() && !levelProcess.hasNext()) { context.setLevelUpperBound(context.getLevel()); } + // if the current memory node is the root if (currentLevel == 0) { + // if all the next level nodes of the root node have not been processed while (context.getLevelUpperBound() != currentLevel) { + // process all pending next-level nodes List children = levelProcess.getChildren(memNode, context); for (O child : children) { - context.setLevel(currentLevel + 1); + // use the processing method of the next layer to process the next layer of nodes levelProcess.getNext().process(child, context); context.setLevel(currentLevel); } + // after each layer is processed, the upper bound is reduced by one context.setLevelUpperBound(context.getLevelUpperBound() - 1); } + // process the current memory node levelProcess.handle(memNode, context); return; } if (currentLevel > context.getLevelUpperBound()) return; + // only process memory nodes with equal level and upper bound if (currentLevel == context.getLevelUpperBound()) { levelProcess.handle(memNode, context); return; } + + // process all pending next-level nodes List children = levelProcess.getChildren(memNode, context); for (O child : children) { context.setLevel(currentLevel + 1); diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java index 6d52ef1577fcd..c1e42f74113b4 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java @@ -21,11 +21,28 @@ import java.io.FileNotFoundException; import java.io.IOException; +/** get records in wal file */ public interface IWALReader { + /** + * close resource + * + * @throws IOException + */ void close() throws IOException; + /** + * determine if there is a next record + * + * @return returns true if there is, else returns false + * @throws FileNotFoundException + */ boolean hasNext() throws FileNotFoundException; + /** + * return the next record + * + * @throws FileNotFoundException + */ WALRecord next() throws FileNotFoundException; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java index 6d3fecb21e2eb..3a7b4e2cda854 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java @@ -20,11 +20,28 @@ import java.io.IOException; +/** write records to wal file */ public interface IWALWriter { + /** + * write walRecord to wal file + * + * @param walRecord record to be written + * @throws IOException + */ void write(WALRecord walRecord) throws IOException; + /** + * force brush + * + * @throws IOException + */ void force() throws IOException; + /** + * close resource + * + * @throws IOException + */ void close() throws IOException; } diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java index 858c2c219dd7d..a5a70b62f1722 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -29,11 +29,15 @@ import java.nio.file.Files; import java.util.NoSuchElementException; +/** get records in wal file */ public class WALReader implements IWALReader { private static final Logger logger = LoggerFactory.getLogger(WALReader.class); + // wal file private final File logFile; + // wal record prototype, clone on read private final WALRecord prototype; private DataInputStream logStream; + // next wal record private WALRecord nextRecord; private boolean fileCorrupted = false; @@ -63,7 +67,9 @@ public boolean hasNext() { if (logSize <= 0) { return false; } + // first clone the object through the prototype nextRecord = prototype.clone(); + // then perform deserialization and assign a value to the new object nextRecord.deserialize(logStream); } catch (EOFException e) { logger.info(e.getMessage()); diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java index 4311b5debfd2a..0aaf77460aab1 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -22,10 +22,22 @@ import java.io.IOException; import java.nio.ByteBuffer; +/** represents a wal record, which can be extended to implement more complex wal records */ public abstract class WALRecord implements Cloneable { + /** + * serialize the wal record + * + * @param buffer byte buffer + */ public abstract void serialize(ByteBuffer buffer); + /** + * deserialize via input stream + * + * @param stream data input stream + * @throws IOException + */ public abstract void deserialize(DataInputStream stream) throws IOException; @Override diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java index abaf89a354dec..5ce97779ce129 100644 --- a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java @@ -29,12 +29,16 @@ import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; +/** write records to wal file */ public class WALWriter implements IWALWriter { private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); + // wal file private File logFile; private FileOutputStream fileOutputStream; private FileChannel channel; + // 4-bit buffer private final ByteBuffer lengthBuffer; + // save wal record serialized byte data private final ByteBuffer walBuffer; private final boolean forceEachWrite; @@ -48,6 +52,12 @@ public WALWriter(File logFile, int walBufferSize, boolean forceEachWrite) walBuffer = ByteBuffer.allocate(walBufferSize); } + /** + * write walRecord to wal file + * + * @param walRecord record to be written + * @throws IOException + */ @Override public void write(WALRecord walRecord) throws IOException { if (channel == null) { diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md index 713e899d49dc8..1b44b95b05038 100644 --- a/schema-engine-tag/README.md +++ b/schema-engine-tag/README.md @@ -23,7 +23,7 @@ # How To Use -Firstly, you should package **schema-engine-rocksdb** by the following command: +Firstly, you should package **schema-engine-tag** by the following command: ```shell mvn clean package -pl schema-engine-tag -am -DskipTests diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java index e78091b9460f9..d360a84327074 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -89,6 +89,7 @@ import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; +/** Mock tag schema region, only used for test */ public class MockTagSchemaRegion implements ISchemaRegion { protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java index 756f0f6f446d4..a47b66cfa0b9e 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java @@ -18,10 +18,13 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion; +/** tag schema region config */ public class TagSchemaConfig { + // the maximum number of device ids managed by a working memTable private int numOfDeviceIdsInMemTable = 65536; + // the size of wal buffer used to store a wal record private int walBufferSize = 1024 * 1024; public int getNumOfDeviceIdsInMemTable() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java index 97f7d4752b99c..881a0e28a722d 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java @@ -31,6 +31,7 @@ import java.io.InputStream; import java.util.Properties; +/** manager tag schema config */ public class TagSchemaDescriptor { private static final Logger logger = LoggerFactory.getLogger(TagSchemaDescriptor.class); @@ -47,9 +48,16 @@ public static TagSchemaDescriptor getInstance() { } private void loadProperties() { - String iotdbHomePath = System.getProperty(IoTDBConstant.IOTDB_HOME, null); + String iotDBHomePath = System.getProperty(IoTDBConstant.IOTDB_HOME, null); + if (iotDBHomePath == null) { + logger.warn( + "Cannot find IOTDB_HOME environment variable when loading " + + "config file {}, use default configuration", + TAG_SCHEMA_CONFIG_FILE_NAME); + return; + } String tagSchemaConfigPath = - iotdbHomePath + iotDBHomePath + File.separatorChar + "conf" + File.separatorChar diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java index 06eed3e5eb51c..93ee6e1540296 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; @@ -96,21 +97,27 @@ import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; +/** tag schema region */ public class TagSchemaRegion implements ISchemaRegion { private static final Logger logger = LoggerFactory.getLogger(TagSchemaRegion.class); protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + // when a path ends with ".**", it represents batch processing private final String TAIL = ".**"; + private final IStorageGroupMNode storageGroupMNode; private final String storageGroupFullPath; private final SchemaRegionId schemaRegionId; private final String schemaRegionDirPath; + // tag inverted index private final TagInvertedIndex tagInvertedIndex; + // manager device id -> INT32 id private final IDeviceIDList deviceIDList; + // manager timeSeries private final IDTable idTable; private final ISeriesNumerLimiter seriesNumerLimiter; @@ -135,6 +142,7 @@ public TagSchemaRegion( @Override public void init() throws MetadataException { + // must enableIDTableLogFile or deviceIDTransformationMethod=="Plain" if (!config.isEnableIDTableLogFile() && config.getDeviceIDTransformationMethod().equals("SHA256")) { throw new MetadataException( @@ -151,11 +159,11 @@ public void init() throws MetadataException { } } } - logger.info("initialized successfully: {}", this); } @Override + @TestOnly public void clear() { try { tagInvertedIndex.clear(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java index 9753cf013dddb..6dda97a00aa1b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java @@ -30,6 +30,7 @@ import java.io.FileOutputStream; import java.io.IOException; +/** for append-only writing of device id list to disk */ public class AppendOnlyDeviceIDListFileManager { private static final Logger logger = @@ -68,17 +69,27 @@ private void initFile() throws IOException { } } - public void serialize(String deviceID) { + /** + * write the device id to file + * + * @param deviceID device id + */ + public void write(String deviceID) { try { if (!isRecover) { ReadWriteIOUtils.write(deviceID, outputStream); } } catch (IOException e) { - logger.error("failed to serialize device id: " + deviceID); - throw new IllegalArgumentException("can't serialize device id of " + deviceID); + logger.error("failed to write device id: " + deviceID); + throw new IllegalArgumentException("can't write device id of " + deviceID); } } + /** + * recover device id list + * + * @param deviceIDList device id list + */ public void recover(DeviceIDList deviceIDList) { logger.info("recover device id list using file {}", deviceIDSFile); try (FileInputStream inputStream = new FileInputStream(deviceIDSFile)) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java index c8793d40fdfa4..4099248724d71 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java @@ -25,8 +25,10 @@ import java.util.ArrayList; import java.util.List; +/** manage device id -> int32 id */ public class DeviceIDList implements IDeviceIDList { + // use an array list to manage device id -> int id private final List deviceIDS; private AppendOnlyDeviceIDListFileManager appendOnlyDeviceIDListFileManager; @@ -44,7 +46,7 @@ public void recover() { @Override public void add(IDeviceID deviceID) { deviceIDS.add(deviceID); - appendOnlyDeviceIDListFileManager.serialize(deviceID.toStringID()); + appendOnlyDeviceIDListFileManager.write(deviceID.toStringID()); } @Override diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java index 1a0d72571a8cd..3d461a4b7097a 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java @@ -24,14 +24,36 @@ import java.io.IOException; import java.util.List; +/** manage device id -> int32 id */ public interface IDeviceIDList { + /** + * insert a device id + * + * @param deviceID device id + */ void add(IDeviceID deviceID); + /** + * get device id using int32 id + * + * @param index int32 id + * @return device id + */ IDeviceID get(int index); + /** + * returns the number of managed device ids + * + * @return the number of managed device ids + */ int size(); + /** + * get all managed device ids + * + * @return all managed device ids + */ List getAllDeviceIDS(); @TestOnly diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java index d5269bbce987e..18c90ada1550e 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -18,21 +18,50 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; -import org.apache.iotdb.lsm.context.DeleteContext; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import java.util.List; import java.util.Map; +/** tag inverted index interface */ public interface ITagInvertedIndex { - void addTags(InsertContext context); + /** + * insert tags and id using insert request context + * + * @param context insert request context + */ + void addTags(InsertRequestContext context); + /** + * insert tags and device id + * + * @param tags tags like: + * @param id INT32 device id + */ void addTags(Map tags, int id); - void removeTags(DeleteContext context); + /** + * delete tags and id using delete request context + * + * @param context delete request context + */ + void removeTags(DeleteRequestContext context); + /** + * delete tags and id using delete request context + * + * @param tags tags like: + * @param id INT32 device id + */ void removeTags(Map tags, int id); + /** + * get all matching device ids + * + * @param tags tags like: + * @return device ids + */ List getMatchedIDs(Map tags); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java index 3ad63e48b8092..e161b5bf93b7d 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; @@ -29,9 +27,9 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover.RecoverManager; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; -import org.apache.iotdb.lsm.context.DeleteContext; -import org.apache.iotdb.lsm.context.InsertContext; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; @@ -45,10 +43,10 @@ import java.util.Map; import java.util.stream.Collectors; +/** tag reverse index implementation class */ public class TagInvertedIndex implements ITagInvertedIndex { - private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class); private static final TagSchemaConfig tagSchemaConfig = TagSchemaDescriptor.getInstance().getTagSchemaConfig(); @@ -63,12 +61,15 @@ public class TagInvertedIndex implements ITagInvertedIndex { private RecoverManager recoverManager; + // the maximum number of device ids managed by a working memTable private int numOfDeviceIdsInMemTable; + // (maxDeviceID / numOfDeviceIdsInMemTable) -> MemTable private Map immutableMemTables; private MemTable workingMemTable; + // the largest device id saved by the current MemTable private int maxDeviceID; public TagInvertedIndex(String schemaDirPath) { @@ -92,9 +93,15 @@ public synchronized void recover() { recoverManager.recover(this); } + /** + * insert tags and id using insert request context + * + * @param context insert request context + */ @Override - public synchronized void addTags(InsertContext context) { + public synchronized void addTags(InsertRequestContext context) { int id = (int) context.getValue(); + // if the device id can not be saved to the current working MemTable if (!inWorkingMemTable(id)) { workingMemTable.setStatus(MemTable.IMMUTABLE); immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); @@ -109,8 +116,15 @@ public synchronized void addTags(InsertContext context) { } } + /** + * insert tags and device id + * + * @param tags tags like: + * @param id INT32 device id + */ @Override public synchronized void addTags(Map tags, int id) { + // if the device id can not be saved to the current working MemTable if (!inWorkingMemTable(id)) { workingMemTable.setStatus(MemTable.IMMUTABLE); immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); @@ -127,8 +141,13 @@ public synchronized void addTags(Map tags, int id) { } } + /** + * delete tags and id using delete request context + * + * @param context delete request context + */ @Override - public void removeTags(DeleteContext context) { + public void removeTags(DeleteRequestContext context) { int id = (int) context.getValue(); MemTable memTable = null; if (inWorkingMemTable(id)) { @@ -143,6 +162,12 @@ public void removeTags(DeleteContext context) { } } + /** + * delete tags and id using delete request context + * + * @param tags tags like: + * @param id INT32 device id + */ @Override public synchronized void removeTags(Map tags, int id) { List memTables = new ArrayList<>(); @@ -160,6 +185,12 @@ public synchronized void removeTags(Map tags, int id) { } } + /** + * get all matching device ids + * + * @param tags tags like: + * @return device ids + */ @Override public synchronized List getMatchedIDs(Map tags) { List memTables = new ArrayList<>(); @@ -194,18 +225,24 @@ public String toString() { + '}'; } + /** + * determine whether the id can be saved to the current MemTable + * + * @param id INT32 device id + * @return return true if it can, otherwise return false + */ private boolean inWorkingMemTable(int id) { return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; } private void addTag(MemTable memTable, String tagKey, String tagValue, int id) throws Exception { - InsertContext insertContext = new InsertContext(id, tagKey, tagValue); + InsertRequestContext insertContext = new InsertRequestContext(id, tagKey, tagValue); insertionManager.process(memTable, insertContext); } private void removeTag(List memTables, String tagKey, String tagValue, int id) throws Exception { - DeleteContext deleteContext = new DeleteContext(id, tagKey, tagValue); + DeleteRequestContext deleteContext = new DeleteRequestContext(id, tagKey, tagValue); for (MemTable memTable : memTables) { deletionManager.process(memTable, deleteContext); } @@ -213,7 +250,7 @@ private void removeTag(List memTables, String tagKey, String tagValue, private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) throws Exception { - QueryContext queryContext = new QueryContext(tagKey, tagValue); + QueryRequestContext queryContext = new QueryRequestContext(tagKey, tagValue); for (MemTable memTable : memTables) { queryManager.process(memTable, queryContext); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java index 211eed0bbe944..51fd8ab031a22 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -20,11 +20,13 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; -public class DeletionManager extends BasicLsmManager { +/** manage deletion to MemTable */ +public class DeletionManager extends BasicLsmManager { + // use wal manager object to write wal file on deletion private WALManager walManager; public DeletionManager(WALManager walManager) { @@ -32,13 +34,21 @@ public DeletionManager(WALManager walManager) { initLevelProcess(); } + /** + * write wal file on deletion + * + * @param root root memory node + * @param context request context + * @throws Exception + */ @Override - public void preProcess(MemTable root, DeleteContext context) throws Exception { + public void preProcess(MemTable root, DeleteRequestContext context) throws Exception { if (!context.isRecover()) { walManager.write(context); } } + /** set the delete operation for each layer of memory nodes */ private void initLevelProcess() { this.nextLevel(new MemTableDeletion()) .nextLevel(new MemChunkGroupDeletion()) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java index c95a617765b66..6b11f648710cb 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java @@ -19,19 +19,34 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; import java.util.List; +/** deletion for MemChunk */ public class MemChunkDeletion extends DeleteLevelProcess { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ @Override - public List getChildren(MemChunk memNode, DeleteContext context) { + public List getChildren(MemChunk memNode, DeleteRequestContext context) { return null; } + /** + * the delete method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context deletion request context + */ @Override - public void delete(MemChunk memNode, DeleteContext context) { + public void delete(MemChunk memNode, DeleteRequestContext context) { Integer deviceID = (Integer) context.getValue(); memNode.remove(deviceID); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java index 651553fd86032..a0f40a069aa93 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java @@ -20,15 +20,24 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; import java.util.ArrayList; import java.util.List; +/** deletion for MemChunkGroup */ public class MemChunkGroupDeletion extends DeleteLevelProcess { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ @Override - public List getChildren(MemChunkGroup memNode, DeleteContext context) { + public List getChildren(MemChunkGroup memNode, DeleteRequestContext context) { List memChunks = new ArrayList<>(); String tagValue = (String) context.getKey(); MemChunk child = memNode.get(tagValue); @@ -36,8 +45,14 @@ public List getChildren(MemChunkGroup memNode, DeleteContext context) return memChunks; } + /** + * the delete method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context deletion request context + */ @Override - public void delete(MemChunkGroup memNode, DeleteContext context) { + public void delete(MemChunkGroup memNode, DeleteRequestContext context) { String tagValue = (String) context.getKey(); MemChunk child = memNode.get(tagValue); if (child == null || child.isEmpty()) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java index 0a1e5b3d05715..4795585b32e57 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java @@ -20,16 +20,25 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.lsm.context.DeleteContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; import java.util.ArrayList; import java.util.List; import java.util.Set; +/** deletion for MemTable */ public class MemTableDeletion extends DeleteLevelProcess { + + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ @Override - public List getChildren(MemTable memNode, DeleteContext context) { + public List getChildren(MemTable memNode, DeleteRequestContext context) { if (memNode.isImmutable()) return new ArrayList<>(); List memChunkGroups = new ArrayList<>(); String tagKey = (String) context.getKey(); @@ -38,8 +47,14 @@ public List getChildren(MemTable memNode, DeleteContext context) return memChunkGroups; } + /** + * the delete method corresponding to the MemTable node + * + * @param memNode memory node + * @param context deletion request context + */ @Override - public void delete(MemTable memNode, DeleteContext context) { + public void delete(MemTable memNode, DeleteRequestContext context) { if (memNode.isImmutable()) { Set deletionList = memNode.getDeletionList(); if (!deletionList.contains(context.getValue())) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java index 0e203a3d646ba..583f8944fd147 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -20,13 +20,15 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; import java.io.IOException; -public class InsertionManager extends BasicLsmManager { +/** manage insertion to MemTable */ +public class InsertionManager extends BasicLsmManager { + // use wal manager object to write wal file on insertion private WALManager walManager; public InsertionManager(WALManager walManager) { @@ -34,13 +36,21 @@ public InsertionManager(WALManager walManager) { initLevelProcess(); } + /** + * write wal file on insertion + * + * @param root root memory node + * @param context insert request context + * @throws Exception + */ @Override - public void preProcess(MemTable root, InsertContext context) throws IOException { + public void preProcess(MemTable root, InsertRequestContext context) throws IOException { if (!context.isRecover()) { walManager.write(context); } } + /** set the insert operation for each layer of memory nodes */ private void initLevelProcess() { this.nextLevel(new MemTableInsertion()) .nextLevel(new MemChunkGroupInsertion()) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java index 843084223f4ea..5877b53f3d46b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java @@ -20,15 +20,24 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; import java.util.ArrayList; import java.util.List; +/** insertion for MemChunkGroup */ public class MemChunkGroupInsertion extends InsertLevelProcess { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ @Override - public List getChildren(MemChunkGroup memNode, InsertContext context) { + public List getChildren(MemChunkGroup memNode, InsertRequestContext context) { List memChunks = new ArrayList<>(); String tagValue = (String) context.getKey(); MemChunk child = memNode.get(tagValue); @@ -36,8 +45,14 @@ public List getChildren(MemChunkGroup memNode, InsertContext context) return memChunks; } + /** + * the insert method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context insert request context + */ @Override - public void insert(MemChunkGroup memNode, InsertContext context) { + public void insert(MemChunkGroup memNode, InsertRequestContext context) { String tagValue = (String) context.getKey(); memNode.put(tagValue); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java index 5dec23dcb7d1b..1cf23bd2e1c04 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java @@ -19,19 +19,34 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; import java.util.List; +/** insertion for MemChunk */ public class MemChunkInsertion extends InsertLevelProcess { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ @Override - public List getChildren(MemChunk memNode, InsertContext context) { + public List getChildren(MemChunk memNode, InsertRequestContext context) { return null; } + /** + * the insert method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context insert request context + */ @Override - public void insert(MemChunk memNode, InsertContext context) { + public void insert(MemChunk memNode, InsertRequestContext context) { Integer deviceID = (Integer) context.getValue(); memNode.put(deviceID); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java index ef88f0591bc03..0d098e88999ab 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java @@ -20,17 +20,24 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; import java.util.ArrayList; import java.util.List; -// memtable的insert操作 +/** insertion for MemTable */ public class MemTableInsertion extends InsertLevelProcess { + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ @Override - public List getChildren(MemTable memNode, InsertContext context) { + public List getChildren(MemTable memNode, InsertRequestContext context) { if (memNode.isImmutable()) return new ArrayList<>(); List memChunkGroups = new ArrayList<>(); String tagKey = (String) context.getKey(); @@ -39,8 +46,14 @@ public List getChildren(MemTable memNode, InsertContext context) return memChunkGroups; } + /** + * the insert method corresponding to the MemTable node + * + * @param memNode memory node + * @param context insert request context + */ @Override - public void insert(MemTable memNode, InsertContext context) { + public void insert(MemTable memNode, InsertRequestContext context) { if (memNode.isImmutable()) return; String tagKey = (String) context.getKey(); memNode.put(tagKey); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java index b46a770ca97e3..80610c1eaec6f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java @@ -20,8 +20,10 @@ import org.roaringbitmap.RoaringBitmap; -// 管理设备id集合 +/** used to manage the device id collection */ public class MemChunk { + + // manage the device id collection, see: https://github.com/RoaringBitmap/RoaringBitmap private RoaringBitmap roaringBitmap; public MemChunk() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java index f6765030183a5..c4dca031639a9 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java @@ -21,7 +21,10 @@ import java.util.HashMap; import java.util.Map; +/** used to manage tagValue -> MemChunk */ public class MemChunkGroup { + + // manage tagValue -> MemChunk private Map memChunkMap; public MemChunkGroup() { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java index f0730e162df1c..5f9cd09c6284b 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -23,15 +23,20 @@ import java.util.Map; import java.util.Set; +/** used to manage tagKey -> MemChunkGroup */ public class MemTable { + public static final String WORKING = "working"; public static final String IMMUTABLE = "immutable"; + // manage tagKey -> MemChunkGroup private Map memChunkGroupMap; private String status; + // if the memTable is immutable, the data cannot be deleted directly, and the deleted data needs + // to be recorded in the deletionList private Set deletionList; public MemTable(String status) { diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java index bf9656e82205d..1ba77a754b540 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java @@ -20,15 +20,24 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; import java.util.ArrayList; import java.util.List; +/** query for MemChunkGroup */ public class MemChunkGroupQuery extends QueryLevelProcess { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ @Override - public List getChildren(MemChunkGroup memNode, QueryContext context) { + public List getChildren(MemChunkGroup memNode, QueryRequestContext context) { List memChunks = new ArrayList<>(); String tagValue = (String) context.getKey(); MemChunk child = memNode.get(tagValue); @@ -36,6 +45,12 @@ public List getChildren(MemChunkGroup memNode, QueryContext context) { return memChunks; } + /** + * the insert method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context query request context + */ @Override - public void query(MemChunkGroup memNode, QueryContext context) {} + public void query(MemChunkGroup memNode, QueryRequestContext context) {} } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java index d67de88f62bb4..d60be13c8431c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java @@ -19,21 +19,36 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; import org.roaringbitmap.RoaringBitmap; import java.util.List; +/** query for MemChunk */ public class MemChunkQuery extends QueryLevelProcess { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ @Override - public List getChildren(MemChunk memNode, QueryContext context) { + public List getChildren(MemChunk memNode, QueryRequestContext context) { return null; } + /** + * the query method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context query request context + */ @Override - public void query(MemChunk memNode, QueryContext context) { + public void query(MemChunk memNode, QueryRequestContext context) { RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); if (roaringBitmap == null) roaringBitmap = new RoaringBitmap(); RoaringBitmap now = RoaringBitmap.or(roaringBitmap, memNode.getRoaringBitmap()); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java index b1acbf845b14e..e5ece3a13a99f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; import org.roaringbitmap.RoaringBitmap; @@ -29,10 +29,18 @@ import java.util.List; import java.util.Set; +/** query for MemTable */ public class MemTableQuery extends QueryLevelProcess { + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ @Override - public List getChildren(MemTable memNode, QueryContext context) { + public List getChildren(MemTable memNode, QueryRequestContext context) { List memChunkGroups = new ArrayList<>(); String tagKey = (String) context.getKey(); MemChunkGroup child = memNode.get(tagKey); @@ -40,9 +48,15 @@ public List getChildren(MemTable memNode, QueryContext context) { return memChunkGroups; } + /** + * the query method corresponding to the MemTable node + * + * @param memNode memory node + * @param context query request context + */ @Override - public void query(MemTable memNode, QueryContext context) { - // 如果是immutable，则需要在查询结果中删除deletionList中的id + public void query(MemTable memNode, QueryRequestContext context) { + // if the memTable is immutable, we need to delete the id in deletionList in the query result if (memNode.isImmutable()) { RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); Set deletionList = memNode.getDeletionList(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java index 4a8e3fdbcb0eb..d3dc3a1d30538 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java @@ -19,15 +19,17 @@ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; -import org.apache.iotdb.lsm.context.QueryContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; import org.apache.iotdb.lsm.manager.BasicLsmManager; -public class QueryManager extends BasicLsmManager { +/** manage insertion to MemTable */ +public class QueryManager extends BasicLsmManager { public QueryManager() { initLevelProcess(); } + /** set the query operation for each layer of memory nodes */ private void initLevelProcess() { this.nextLevel(new MemTableQuery()) .nextLevel(new MemChunkGroupQuery()) diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java index 7cc0015626030..5c697e1c77b70 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -20,13 +20,14 @@ import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.ITagInvertedIndex; import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; -import org.apache.iotdb.lsm.context.Context; -import org.apache.iotdb.lsm.context.DeleteContext; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.context.RequestContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** for memory structure recovery */ public class RecoverManager { private static final Logger logger = LoggerFactory.getLogger(RecoverManager.class); @@ -36,16 +37,21 @@ public RecoverManager(WALManager walManager) { this.walManager = walManager; } + /** + * recover tagInvertedIndex + * + * @param tagInvertedIndex tag inverted index + */ public void recover(ITagInvertedIndex tagInvertedIndex) { logger.info("recover tagInvertedIndex"); while (true) { - Context context = walManager.read(); + RequestContext context = walManager.read(); switch (context.getType()) { case INSERT: - tagInvertedIndex.addTags((InsertContext) context); + tagInvertedIndex.addTags((InsertRequestContext) context); break; case DELETE: - tagInvertedIndex.removeTags((DeleteContext) context); + tagInvertedIndex.removeTags((DeleteRequestContext) context); break; default: return; diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java index 6c14934ce1062..5d73ed74ae60c 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -27,11 +27,16 @@ import java.util.ArrayList; import java.util.List; +/** represents a record in the wal file */ public class WALEntry extends WALRecord { + + // can be insertion(1) or deletion(2) private int type; + // keys at each level private List keys; + // device id private int deviceID; public WALEntry() { @@ -45,6 +50,11 @@ public WALEntry(int type, List keys, int deviceID) { this.deviceID = deviceID; } + /** + * serialize the wal entry + * + * @param buffer byte buffer + */ @Override public void serialize(ByteBuffer buffer) { ReadWriteIOUtils.write(type, buffer); @@ -55,6 +65,12 @@ public void serialize(ByteBuffer buffer) { } } + /** + * deserialize from DataInputStream + * + * @param stream data input stream + * @throws IOException + */ @Override public void deserialize(DataInputStream stream) throws IOException { this.type = stream.readInt(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java index 65a74bba89144..e5047a0233b6e 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -18,11 +18,12 @@ */ package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; -import org.apache.iotdb.lsm.context.Context; -import org.apache.iotdb.lsm.context.DeleteContext; -import org.apache.iotdb.lsm.context.InsertContext; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.context.RequestContext; import org.apache.iotdb.lsm.wal.WALReader; import org.apache.iotdb.lsm.wal.WALWriter; @@ -31,7 +32,9 @@ import java.util.ArrayList; import java.util.List; +/** Manage wal entry writes and reads */ public class WALManager { + private static final String WAL_FILE_NAME = "tag_inverted_index.log"; private static final int INSERT = 1; @@ -45,8 +48,10 @@ public class WALManager { private File walFile; + // directly use the wal writer that comes with the lsm framework private WALWriter walWriter; + // directly use the wal reader that comes with the lsm framework private WALReader walReader; public WALManager(String schemaDirPath) throws IOException { @@ -66,21 +71,31 @@ private void initFile(String schemaDirPath) throws IOException { } } - public synchronized void write(Context context) throws IOException { + /** + * handle wal log writes for each request context + * + * @param context request context + * @throws IOException + */ + public synchronized void write(RequestContext context) throws IOException { switch (context.getType()) { case INSERT: - process((InsertContext) context); + process((InsertRequestContext) context); break; case DELETE: - process((DeleteContext) context); + process((DeleteRequestContext) context); break; default: break; } } - // 用于recover - public synchronized Context read() { + /** + * for recover + * + * @return request context + */ + public synchronized RequestContext read() { if (walReader.hasNext()) { WALEntry walEntry = (WALEntry) walReader.next(); if (walEntry.getType() == INSERT) { @@ -90,11 +105,17 @@ public synchronized Context read() { return generateDeleteContext(walEntry); } } - return new Context(); + return new RequestContext(); } - private InsertContext generateInsertContext(WALEntry walEntry) { - InsertContext insertContext = new InsertContext(); + /** + * generate insert context from wal entry + * + * @param walEntry wal entry + * @return insert context + */ + private InsertRequestContext generateInsertContext(WALEntry walEntry) { + InsertRequestContext insertContext = new InsertRequestContext(); List objects = new ArrayList<>(); objects.addAll(walEntry.getKeys()); insertContext.setKeys(objects); @@ -103,8 +124,15 @@ private InsertContext generateInsertContext(WALEntry walEntry) { return insertContext; } - private DeleteContext generateDeleteContext(WALEntry walEntry) { - DeleteContext deleteContext = new DeleteContext(walEntry.getDeviceID(), walEntry.getKeys()); + /** + * generate delete context from wal entry + * + * @param walEntry wal entry + * @return delete context + */ + private DeleteRequestContext generateDeleteContext(WALEntry walEntry) { + DeleteRequestContext deleteContext = + new DeleteRequestContext(walEntry.getDeviceID(), walEntry.getKeys()); List objects = new ArrayList<>(); objects.addAll(walEntry.getKeys()); deleteContext.setKeys(objects); @@ -113,7 +141,13 @@ private DeleteContext generateDeleteContext(WALEntry walEntry) { return deleteContext; } - private void process(InsertContext insertContext) throws IOException { + /** + * handle wal log writes for each insert context + * + * @param insertContext insert context + * @throws IOException + */ + private void process(InsertRequestContext insertContext) throws IOException { List objects = insertContext.getKeys(); List keys = new ArrayList<>(); for (Object o : objects) { @@ -123,7 +157,13 @@ private void process(InsertContext insertContext) throws IOException { walWriter.write(walEntry); } - private void process(DeleteContext deleteContext) throws IOException { + /** + * handle wal log writes for each delete context + * + * @param deleteContext delete context + * @throws IOException + */ + private void process(DeleteRequestContext deleteContext) throws IOException { List objects = deleteContext.getKeys(); List keys = new ArrayList<>(); for (Object o : objects) { @@ -133,6 +173,7 @@ private void process(DeleteContext deleteContext) throws IOException { walWriter.write(walEntry); } + @TestOnly public void close() throws IOException { walWriter.close(); walReader.close(); diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java index 5c792000d0b9b..b6f5402a7169f 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java @@ -28,7 +28,19 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +/** process MeasurementPath */ public class MeasurementPathUtils { + + /** + * generate MeasurementPath + * + * @param devicePath device path + * @param measurement measurement + * @param schemaEntry schema entry + * @param isAligned is aligned + * @return MeasurementPath + * @throws IllegalPathException + */ public static MeasurementPath generateMeasurementPath( String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) throws IllegalPathException { @@ -45,6 +57,13 @@ public static MeasurementPath generateMeasurementPath( return measurementPath; } + /** + * generate MeasurementPath + * + * @param diskSchemaEntry disk schema entry + * @return MeasurementPath + * @throws IllegalPathException + */ public static MeasurementPath generateMeasurementPath(DiskSchemaEntry diskSchemaEntry) throws IllegalPathException { MeasurementPath measurementPath = diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java index 59fbc6409ed6f..59208fc418b67 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java @@ -21,8 +21,16 @@ import java.util.Map; import java.util.TreeMap; +/** path and tag converter */ public class PathTagConverterUtils { + /** + * convert the path of the tree model to the tags of the tag model + * + * @param storageGroupFullPath storage group full path + * @param path path of the tree model + * @return tags of the tag model + */ public static Map pathToTags(String storageGroupFullPath, String path) { if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); String devicePath = path.substring(storageGroupFullPath.length() + 1); @@ -34,6 +42,13 @@ public static Map pathToTags(String storageGroupFullPath, String return tagsMap; } + /** + * convert the tags of the tag model to the path of the tree model + * + * @param storageGroupFullPath storage group full path + * @param tags tags of the tag model + * @return path of the tree model + */ public static String tagsToPath(String storageGroupFullPath, Map tags) { StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); for (String tagKey : tags.keySet()) { @@ -42,6 +57,13 @@ public static String tagsToPath(String storageGroupFullPath, Map return stringBuilder.toString(); } + /** + * generate unique path for paths with the same semantics + * + * @param storageGroupFullPath storage group full path + * @param path path of the tree model + * @return unique path of the tree model + */ public static String pathToTagsSortPath(String storageGroupFullPath, String path) { return tagsToPath(storageGroupFullPath, pathToTags(storageGroupFullPath, path)); } diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java index 14f02a7dfd205..18d4c22662194 100644 --- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java @@ -23,7 +23,18 @@ import java.util.HashMap; +/** process show timeSeries result */ public class ShowTimeSeriesResultUtils { + + /** + * generate show timeSeries result + * + * @param sgName storage group name + * @param devicePath device path + * @param measurement measurement + * @param schemaEntry schema entry + * @return ShowTimeSeriesResult + */ public static ShowTimeSeriesResult generateShowTimeSeriesResult( String sgName, String devicePath, String measurement, SchemaEntry schemaEntry) { return new ShowTimeSeriesResult( @@ -38,6 +49,14 @@ public static ShowTimeSeriesResult generateShowTimeSeriesResult( new HashMap<>()); } + /** + * generate show timeSeries result + * + * @param sgName storage group name + * @param timeSeriesPath timeSeries path + * @param schemaEntry schema entry + * @return ShowTimeSeriesResult + */ public static ShowTimeSeriesResult generateShowTimeSeriesResult( String sgName, String timeSeriesPath, SchemaEntry schemaEntry) { return new ShowTimeSeriesResult( From 60038d52a8c69707a18cb34cb23526f3b315989b Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 23:04:05 +0800 Subject: [PATCH 28/53] add README.md --- schema-engine-tag/README.md | 156 +++++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 3 deletions(-) diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md index 1b44b95b05038..0afdc21b0ae5c 100644 --- a/schema-engine-tag/README.md +++ b/schema-engine-tag/README.md @@ -19,7 +19,16 @@ --> -`TagSchemaRegion` is an implementation of `SchemaRegion`. +
+  _____             _____      _                           ______           _             
+|_   _|           /  ___|    | |                          | ___ \         (_)            
+  | | __ _  __ _  \ `--.  ___| |__   ___ _ __ ___   __ _  | |_/ /___  __ _ _  ___  _ __  
+  | |/ _` |/ _` |  `--. \/ __| '_ \ / _ \ '_ ` _ \ / _` | |    // _ \/ _` | |/ _ \| '_ \ 
+  | | (_| | (_| | /\__/ / (__| | | |  __/ | | | | | (_| | | |\ \  __/ (_| | | (_) | | | |
+  \_/\__,_|\__, | \____/ \___|_| |_|\___|_| |_| |_|\__,_| \_| \_\___|\__, |_|\___/|_| |_|
+            __/ |                                                     __/ |              
+           |___/                                                     |___/ > version 0.14.0-SNAPSHOT
+
# How To Use @@ -34,5 +43,146 @@ schema-engine-tag/target/schema-engine-tag. Copy the file in the conf directory and copy the files in the lib directory to the lib directory of server. Then, open the **iotdb-datanode.properties** in the conf directory of server, and set the `schema_engine_mode` to -Tag, set the `enable_id_table` to true. Restart the IoTDB, the system will use `TagSchemaRegion` to manage -the metadata. \ No newline at end of file +**Tag**, set the `enable_id_table` to **true**. Restart the IoTDB, the system will use `TagSchemaRegion` to manage +the metadata. + +## Use Cli + +IoTDB offers different ways to interact with server, here we introduce the basic steps of using Cli tool to insert and query data. +The command line cli is interactive, so you should see the welcome logo and statements if everything is ready: +```txt +--------------------- +Starting IoTDB Cli +--------------------- + _____ _________ ______ ______ +|_ _| | _ _ ||_ _ `.|_ _ \ + | | .--.|_/ | | \_| | | `. \ | |_) | + | | / .'`\ \ | | | | | | | __'. + _| |_| \__. | _| |_ _| |_.' /_| |__) | +|_____|'.__.' |_____| |______.'|_______/ version 0.14.0-SNAPSHOT + + +IoTDB> login successfully +``` +### create timeseries + +- create timeseries + +```txt +IoTDB> create timeseries root.ln.tag1.a.tag2.b.status with datatype=BOOLEAN,encoding=PLAIN +Msg: The statement is executed successfully. +``` +- create aligned timeseries + +```txt +IoTDB> CREATE ALIGNED TIMESERIES root.ln.tag1.a.tag2.c(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT encoding=PLAIN compressor=SNAPPY) + +Msg: The statement is executed successfully. +``` + +### show timeserie + +- point query + +enter a full path + +```txt +IoTDB> show timeseries root.ln.tag2.c.tag1.a ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +``` + +- batch query + +paths ending in ".**" indicate batch query + +```txt +IoTDB> show timeseries root.ln.tag1.a.** ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.b.status| null| root.ln| BOOLEAN| PLAIN| SNAPPY|null| null| +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ + +IoTDB> show timeseries root.ln.tag2.c.** ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ + +IoTDB> show timeseries root.ln.tag2.b.** ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +|root.ln.tag1.a.tag2.b.status| null| root.ln| BOOLEAN| PLAIN| SNAPPY|null| null| ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +``` + +### insert + +- insert a single column of data + +```txt +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(1,true) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(2,false) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(3,true) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag1.a.tag2.d(timestamp,status) values(1,true) +Msg: The statement is executed successfully. +``` + +- insert alignment data + +```txt +IoTDB> insert into root.sg1.tag1.a(time, s1, s2) aligned values(2, 2, 2), (3, 3, 3) +Msg: The statement is executed successfully. +``` + +### select + +- point query + +```txt +IoTDB> select * from root.sg1.tag1.a ++-----------------------------+------------------+------------------+ +| Time|root.sg1.tag1.a.s1|root.sg1.tag1.a.s2| ++-----------------------------+------------------+------------------+ +|1970-01-01T08:00:00.002+08:00| 2.0| 2.0| +|1970-01-01T08:00:00.003+08:00| 3.0| 3.0| ++-----------------------------+------------------+------------------+ +``` + +- align by device + +```txt +IoTDB> select * from root.sg1.tag1.a align by device ++-----------------------------+---------------+---+---+ +| Time| Device| s1| s2| ++-----------------------------+---------------+---+---+ +|1970-01-01T08:00:00.002+08:00|root.sg1.tag1.a|2.0|2.0| +|1970-01-01T08:00:00.003+08:00|root.sg1.tag1.a|3.0|3.0| ++-----------------------------+---------------+---+---+ +``` + +- batch query + +```txt +IoTDB> select status from root.ln.tag2.d.** where time < 2017-11-01T00:08:00.000 ++-----------------------------+----------------------------+---------------------+ +| Time|root.ln.tag1.a.tag2.d.status|root.ln.tag2.d.status| ++-----------------------------+----------------------------+---------------------+ +|1970-01-01T08:00:00.001+08:00| true| true| +|1970-01-01T08:00:00.002+08:00| null| false| +|1970-01-01T08:00:00.003+08:00| null| true| ++-----------------------------+----------------------------+---------------------+ +``` \ No newline at end of file From b2f5bfdb52e6be849c0b9644f576d228e34ae1c0 Mon Sep 17 00:00:00 2001 From: KeePromMise Date: Fri, 23 Sep 2022 23:07:29 +0800 Subject: [PATCH 29/53] add README.md --- schema-engine-tag/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md index 0afdc21b0ae5c..a00fe79b8d49e 100644 --- a/schema-engine-tag/README.md +++ b/schema-engine-tag/README.md @@ -19,6 +19,8 @@ --> +# Tag Schema Region +`TagSchemaRegion` is an implementation of `SchemaRegion` 
   _____             _____      _                           ______           _             
 |_   _|           /  ___|    | |                          | ___ \         (_)            

From f6fe85d69f52c9de6096754059f6ad2309558359 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Fri, 23 Sep 2022 23:10:33 +0800
Subject: [PATCH 30/53] add README.md

---
 schema-engine-tag/README.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md
index a00fe79b8d49e..e4d70f9e1502e 100644
--- a/schema-engine-tag/README.md
+++ b/schema-engine-tag/README.md
@@ -52,7 +52,7 @@ the metadata.
 
 IoTDB offers different ways to interact with server, here we introduce the basic steps of using Cli tool to insert and query data.
 The command line cli is interactive, so you should see the welcome logo and statements if everything is ready:
-```txt
+```sql
 ---------------------
 Starting IoTDB Cli
 ---------------------
@@ -70,13 +70,13 @@ IoTDB> login successfully
 
 - create timeseries
 
-```txt
+```sql
 IoTDB> create timeseries root.ln.tag1.a.tag2.b.status with datatype=BOOLEAN,encoding=PLAIN
 Msg: The statement is executed successfully.
 ```
 - create aligned timeseries
 
-```txt
+```sql
 IoTDB> CREATE ALIGNED TIMESERIES root.ln.tag1.a.tag2.c(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT  encoding=PLAIN compressor=SNAPPY)
 
 Msg: The statement is executed successfully.
@@ -88,7 +88,7 @@ Msg: The statement is executed successfully.
 
 enter a full path
 
-```txt
+```sql
 IoTDB> show timeseries root.ln.tag2.c.tag1.a
 +-------------------------------+-----+-------------+--------+--------+-----------+----+----------+
 |                     timeseries|alias|storage group|dataType|encoding|compression|tags|attributes|
@@ -102,7 +102,7 @@ IoTDB> show timeseries root.ln.tag2.c.tag1.a
 
 paths ending in ".**" indicate batch query
 
-```txt
+```sql
 IoTDB> show timeseries root.ln.tag1.a.**
 +-------------------------------+-----+-------------+--------+--------+-----------+----+----------+
 |                     timeseries|alias|storage group|dataType|encoding|compression|tags|attributes|
@@ -132,7 +132,7 @@ IoTDB> show timeseries root.ln.tag2.b.**
 
 - insert a single column of data
 
-```txt
+```sql
 IoTDB> insert into root.ln.tag2.d(timestamp,status) values(1,true)
 Msg: The statement is executed successfully.
 IoTDB> insert into root.ln.tag2.d(timestamp,status) values(2,false)
@@ -145,7 +145,7 @@ Msg: The statement is executed successfully.
 
 - insert alignment data
 
-```txt
+```sql
 IoTDB> insert into root.sg1.tag1.a(time, s1, s2) aligned values(2, 2, 2), (3, 3, 3)
 Msg: The statement is executed successfully.
 ```
@@ -154,7 +154,7 @@ Msg: The statement is executed successfully.
 
 - point query
 
-```txt
+```sql
 IoTDB> select * from root.sg1.tag1.a
 +-----------------------------+------------------+------------------+
 |                         Time|root.sg1.tag1.a.s1|root.sg1.tag1.a.s2|
@@ -166,7 +166,7 @@ IoTDB> select * from root.sg1.tag1.a
 
 - align by device
 
-```txt
+```sql
 IoTDB> select * from root.sg1.tag1.a align by device
 +-----------------------------+---------------+---+---+
 |                         Time|         Device| s1| s2|
@@ -178,7 +178,7 @@ IoTDB> select * from root.sg1.tag1.a align by device
 
 - batch query
 
-```txt
+```sql
 IoTDB> select status from root.ln.tag2.d.** where time < 2017-11-01T00:08:00.000
 +-----------------------------+----------------------------+---------------------+
 |                         Time|root.ln.tag1.a.tag2.d.status|root.ln.tag2.d.status|

From e77eadf1c961256241691c69c0341c20dd7484ec Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sat, 15 Oct 2022 10:28:54 +0800
Subject: [PATCH 31/53] move lsm module to tag schema region

---
 lsm/README.md                                 | 396 ------------------
 lsm/pom.xml                                   |  32 --
 pom.xml                                       |   1 -
 .../tagSchemaRegion/TagSchemaRegion.java      |   5 +-
 .../{ => config}/TagSchemaConfig.java         |   2 +-
 .../{ => config}/TagSchemaDescriptor.java     |   2 +-
 .../tagIndex/TagInvertedIndex.java            |   4 +-
 .../tagIndex/wal/WALManager.java              |   4 +-
 .../lsm/context/DeleteRequestContext.java     |   0
 .../lsm/context/FlushRequestContext.java      |   0
 .../lsm/context/InsertRequestContext.java     |   0
 .../lsm/context/QueryRequestContext.java      |   0
 .../iotdb/lsm/context/RequestContext.java     |   0
 .../apache/iotdb/lsm/context/RequestType.java |   0
 .../lsm/levelProcess/BasicLevelProcess.java   |   0
 .../lsm/levelProcess/DeleteLevelProcess.java  |   0
 .../lsm/levelProcess/FlushLevelProcess.java   |   0
 .../lsm/levelProcess/InsertLevelProcess.java  |   0
 .../iotdb/lsm/levelProcess/LevelProcess.java  |   0
 .../lsm/levelProcess/QueryLevelProcess.java   |   0
 .../iotdb/lsm/manager/BasicLsmManager.java    |   0
 .../apache/iotdb/lsm/manager/LsmManager.java  |   0
 .../iotdb/lsm/strategy/AccessStrategy.java    |   0
 .../iotdb/lsm/strategy/BFSAccessStrategy.java |   0
 .../lsm/strategy/PostOrderAccessStrategy.java |   0
 .../lsm/strategy/PreOrderAccessStrategy.java  |   0
 .../lsm/strategy/RBFSAccessStrategy.java      |   0
 .../org/apache/iotdb/lsm/wal/IWALReader.java  |   0
 .../org/apache/iotdb/lsm/wal/IWALWriter.java  |   0
 .../org/apache/iotdb/lsm/wal/WALReader.java   |   0
 .../org/apache/iotdb/lsm/wal/WALRecord.java   |   0
 .../org/apache/iotdb/lsm/wal/WALWriter.java   |   0
 .../tagIndex/TagInvertedIndexTest.java        |   2 +-
 33 files changed, 10 insertions(+), 438 deletions(-)
 delete mode 100644 lsm/README.md
 delete mode 100644 lsm/pom.xml
 rename schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/{ => config}/TagSchemaConfig.java (96%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/{ => config}/TagSchemaDescriptor.java (98%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/context/RequestType.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java (100%)
 rename {lsm => schema-engine-tag}/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java (100%)

diff --git a/lsm/README.md b/lsm/README.md
deleted file mode 100644
index d1c2ae75975fb..0000000000000
--- a/lsm/README.md
+++ /dev/null
@@ -1,396 +0,0 @@
-
-
-# LSM Document
-
-  _____      ______   ____    ____  
- |_   _|   .' ____ \ |_   \  /   _| 
-   | |     | (___ \_|  |   \/   |   
-   | |   _  _.____`.   | |\  /| |   
-  _| |__/ || \____) | _| |_\/_| |_  
- |________| \______.'|_____||_____| > version 0.14.0-SNAPSHOT
-                                    
-

-## Abstract
-
-The lsm framework has implemented the **memory structure** at present, and users only need to define the memory structure and access method of each level.
-
-
-## Example
-Suppose we need to implement an lsm storage engine to store each record similar to , we define the following four layers of memory nodes.
-### Memory structure
-Implement the memory structure of each layer
-
-- MemChunk
-
-The last layer of memory nodes, save the id list
-```java
-public class MemChunk {
-	List deviceIDS;
-
-	public MemChunk() {
-		deviceIDS = new ArrayList<>();
-	}
-
-	public List getDeviceIDS() {
-		return deviceIDS;
-	}
-
-	public void setDeviceIDS(List deviceIDS) {
-		this.deviceIDS = deviceIDS;
-	}
-
-	@Override
-	public String toString() {
-		return "MemChunk{" + deviceIDS.toString() + '}';
-	}
-}
-```
-
-- MemGroup
-
-Use a map to manage tagValue->MemChunk
-```java
-public class MemGroup {
-
-	Map map;
-
-	public MemGroup() {
-		map = new HashMap<>();
-	}
-
-	public Map getMap() {
-		return map;
-	}
-
-	public void setMap(Map map) {
-		this.map = map;
-	}
-
-	@Override
-	public String toString() {
-		return "MemGroup{" + map.toString() + '}';
-	}
-}
-```
-
-- MemTable
-
-Use a map to manage tagKey->MemGroup
-```java
-public class MemTable {
-
-	private Map map;
-
-	public MemTable() {
-		map = new HashMap<>();
-	}
-
-	public Map getMap() {
-		return map;
-	}
-
-	public void setMap(Map map) {
-		this.map = map;
-	}
-
-	@Override
-	public String toString() {
-		return "MemTable{" + map.toString() + '}';
-	}
-}
-```
-
-- MemTableManager
-
-Manage working memTable, and immutable memTable
-```java
-public class MemTableManager {
-
-	private MemTable working;
-
-	private Map immutables;
-
-	private int maxDeviceID;
-
-	public MemTableManager() {
-		working = new MemTable();
-		immutables = new HashMap<>();
-		maxDeviceID = 0;
-	}
-
-	public MemTable getWorking() {
-		return working;
-	}
-
-	public void setWorking(MemTable working) {
-		this.working = working;
-	}
-
-	public Map getImmutables() {
-		return immutables;
-	}
-
-	public void setImmutables(Map immutables) {
-		this.immutables = immutables;
-	}
-
-	public int getMaxDeviceID() {
-		return maxDeviceID;
-	}
-
-	public void setMaxDeviceID(int maxDeviceID) {
-		this.maxDeviceID = maxDeviceID;
-	}
-
-	@Override
-	public String toString() {
-		return "MemTableManager{"
-				+ "working="
-				+ working.toString()
-				+ ", immutables="
-				+ immutables.toString()
-				+ '}';
-	}
-}
-```
-
-### Access method
-Incoming access to each layer for the framework
-
-- Insertion and flush example
-```java
-public class Main {
-	public static void main(String[] args) throws Exception {
-		MemTableManager memTableManager = new MemTableManager();
-		System.out.println("-------------insert--------------");
-		insertionExample(memTableManager);
-		System.out.println("-------------flush--------------");
-		flushExample(memTableManager);
-	}
-    
-	public static void insertionExample(MemTableManager memTableManager) throws Exception {
-        // Initialize a BasicLsmManager to manage insert operations
-		BasicLsmManager baseLsmManager =
-				new BasicLsmManager();
-		baseLsmManager
-				.nextLevel(
-                        // The insert method of the MemTableManager level
-                        new InsertLevelProcess() {
-							@Override
-							public List getChildren(
-									MemTableManager memNode, InsertRequestContext context) {
-								Integer deviceID = (Integer) context.getValue();
-								int maxDeviceID = memNode.getMaxDeviceID();
-								List children = new ArrayList<>();
-								if (deviceID / 65536 == maxDeviceID / 65536) {
-									children.add(memNode.getWorking());
-								} else {
-									children.add(memNode.getImmutables().get(deviceID / 65536));
-								}
-								return children;
-							}
-
-							@Override
-							public void insert(MemTableManager memNode, InsertRequestContext context) {
-								Integer deviceID = (Integer) context.getValue();
-								int maxDeviceID = memNode.getMaxDeviceID();
-								if (deviceID / 65536 == maxDeviceID / 65536) {
-									if (memNode.getWorking() == null) {
-										memNode.setWorking(new MemTable());
-									}
-								} else if (deviceID > maxDeviceID) {
-									memNode
-											.getImmutables()
-											.put(memNode.getMaxDeviceID() / 65536, memNode.getWorking());
-									memNode.setWorking(new MemTable());
-								}
-								if (deviceID > maxDeviceID) {
-									memNode.setMaxDeviceID(deviceID);
-								}
-							}
-						})
-				.nextLevel(
-                        // The insert method of the MemTable level
-						new InsertLevelProcess() {
-							@Override
-							public List getChildren(MemTable memNode, InsertRequestContext context) {
-								String key = (String) context.getKey();
-								List children = new ArrayList<>();
-								children.add(memNode.getMap().get(key));
-								return children;
-							}
-
-							@Override
-							public void insert(MemTable memNode, InsertRequestContext context) {
-								String key = (String) context.getKey();
-								Map map = memNode.getMap();
-								if (map.containsKey(key)) return;
-								map.put(key, new MemGroup());
-							}
-						})
-				.nextLevel(
-                        // The insert method of the MemGroup level
-						new InsertLevelProcess() {
-							@Override
-							public List getChildren(MemGroup memNode, InsertRequestContext context) {
-								String key = (String) context.getKey();
-								List children = new ArrayList<>();
-								children.add(memNode.getMap().get(key));
-								return children;
-							}
-
-							@Override
-							public void insert(MemGroup memNode, InsertRequestContext context) {
-								String key = (String) context.getKey();
-								Map map = memNode.getMap();
-								if (map.containsKey(key)) return;
-								map.put(key, new MemChunk());
-							}
-						})
-				.nextLevel(
-                        // The insert method of the MemChunk level
-						new InsertLevelProcess() {
-							@Override
-							public List getChildren(MemChunk memNode, InsertRequestContext context) {
-								return null;
-							}
-
-							@Override
-							public void insert(MemChunk memNode, InsertRequestContext context) {
-								Integer deviceID = (Integer) context.getValue();
-								List deviceIDs = memNode.getDeviceIDS();
-								deviceIDs.add(deviceID);
-							}
-						});
-        
-        // Insert some  records
-        // The key at the MemTableManager level is null
-		baseLsmManager.process(memTableManager, new InsertRequestContext(1, null, "a", "b"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(2, null, "a", "d"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(3, null, "a", "e"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(4, null, "a", "b"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(5, null, "a1", "b"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(6, null, "a2", "b"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(65535, null, "a", "b"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(65536, null, "a", "b"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(2, null, "a", "d"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(3, null, "a", "e"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(4, null, "a", "b"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(5, null, "a1", "b"));
-		baseLsmManager.process(memTableManager, new InsertRequestContext(6, null, "a2", "b"));
-        // process memTableManager
-		System.out.println(memTableManager);
-	}
-
-	public static void flushExample(MemTableManager memTableManager) throws Exception {
-        // Initialize a BasicLsmManager to manage insert operations
-		BasicLsmManager flushManager =
-				new BasicLsmManager();
-
-		flushManager
-				.nextLevel(
-                        // The insert method of the MemTableManager level
-						new FlushLevelProcess() {
-							@Override
-							public void flush(MemTableManager memNode, FlushRequestContext context) {
-								System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]");
-							}
-
-							@Override
-							public List getChildren(
-									MemTableManager memNode, FlushRequestContext context) {
-								List memTables = new ArrayList<>();
-								memTables.addAll(memNode.getImmutables().values());
-								if (memNode.getWorking() != null) memTables.add(memNode.getWorking());
-								return memTables;
-							}
-						})
-				.nextLevel(
-                        // The insert method of the MemTable level
-						new FlushLevelProcess() {
-							@Override
-							public void flush(MemTable memNode, FlushRequestContext context) {
-								System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]");
-							}
-
-							@Override
-							public List getChildren(MemTable memNode, FlushRequestContext context) {
-								List memGroups = new ArrayList<>();
-								memGroups.addAll(memNode.getMap().values());
-								return memGroups;
-							}
-						})
-				.nextLevel(
-                        // The insert method of the MemGroup level
-						new FlushLevelProcess() {
-							@Override
-							public void flush(MemGroup memNode, FlushRequestContext context) {
-								System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]");
-							}
-
-							@Override
-							public List getChildren(MemGroup memNode, FlushRequestContext context) {
-								List memChunk = new ArrayList<>();
-								memChunk.addAll(memNode.getMap().values());
-								return memChunk;
-							}
-						})
-				.nextLevel(
-                        // The insert method of the MemChunk level
-						new FlushLevelProcess() {
-							@Override
-							public void flush(MemChunk memNode, FlushRequestContext context) {
-								System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]");
-							}
-
-							@Override
-							public List getChildren(MemChunk memNode, FlushRequestContext context) {
-								return new ArrayList<>();
-							}
-						});
-        
-        // process memTableManager
-		flushManager.process(memTableManager, new FlushRequestContext());
-	}
-}
-
-```
-- Output
-
-```txt
--------------insert--------------
-MemTableManager{working=MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}, immutables={0=MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}}}
--------------flush--------------
-FLUSH: MemChunk{[5, 5]}-->[level:3]
-FLUSH: MemChunk{[1, 4, 65535, 4]}-->[level:3]
-FLUSH: MemChunk{[2, 2]}-->[level:3]
-FLUSH: MemChunk{[3, 3]}-->[level:3]
-FLUSH: MemChunk{[6, 6]}-->[level:3]
-FLUSH: MemChunk{[65536]}-->[level:3]
-FLUSH: MemGroup{{b=MemChunk{[5, 5]}}}-->[level:2]
-FLUSH: MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}-->[level:2]
-FLUSH: MemGroup{{b=MemChunk{[6, 6]}}}-->[level:2]
-FLUSH: MemGroup{{b=MemChunk{[65536]}}}-->[level:2]
-FLUSH: MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}-->[level:1]
-FLUSH: MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}-->[level:1]
-FLUSH: MemTableManager{working=MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}, immutables={0=MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}}}-->[level:0]
-```
diff --git a/lsm/pom.xml b/lsm/pom.xml
deleted file mode 100644
index b95e98aea9d2b..0000000000000
--- a/lsm/pom.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-
-
-
-    
-        iotdb-parent
-        org.apache.iotdb
-        0.14.0-SNAPSHOT
-        ../pom.xml
-    
-    4.0.0
-    iotdb-lsm
-    IoTDB lsm
-
diff --git a/pom.xml b/pom.xml
index 6bb952ace414d..5f19e9a9ef97c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -121,7 +121,6 @@
         trigger-api
         rewrite-tsfile-tool
         external-api
-        lsm
         schema-engine-tag
     
     
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
index 93ee6e1540296..be7a58511bc36 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
@@ -35,7 +35,7 @@
 import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException;
 import org.apache.iotdb.db.metadata.LocalSchemaProcessor;
 import org.apache.iotdb.db.metadata.idtable.IDTable;
-import org.apache.iotdb.db.metadata.idtable.IDTableManager;
+import org.apache.iotdb.db.metadata.idtable.IDTableHashmapImpl;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
 import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry;
@@ -134,7 +134,8 @@ public TagSchemaRegion(
     schemaRegionDirPath = storageGroupDirPath + File.separator + schemaRegionId.getId();
     this.storageGroupMNode = storageGroupMNode;
     this.seriesNumerLimiter = seriesNumerLimiter;
-    idTable = IDTableManager.getInstance().getIDTable(storageGroup);
+    File schemaRegionDir = new File(schemaRegionDirPath);
+    idTable = new IDTableHashmapImpl(schemaRegionDir);
     tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath);
     deviceIDList = new DeviceIDList(schemaRegionDirPath);
     init();
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaConfig.java
similarity index 96%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaConfig.java
index a47b66cfa0b9e..74e5bb41fd984 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaConfig.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.db.metadata.tagSchemaRegion;
+package org.apache.iotdb.db.metadata.tagSchemaRegion.config;
 
 /** tag schema region config */
 public class TagSchemaConfig {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaDescriptor.java
similarity index 98%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaDescriptor.java
index 881a0e28a722d..f6b96de87d009 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaDescriptor.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.db.metadata.tagSchemaRegion;
+package org.apache.iotdb.db.metadata.tagSchemaRegion.config;
 
 import org.apache.iotdb.commons.conf.IoTDBConstant;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index e161b5bf93b7d..a40decbf786c4 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -19,8 +19,8 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
 
 import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
index e5047a0233b6e..2c59777b7dd28 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
@@ -19,8 +19,8 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal;
 
 import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.context.RequestContext;
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestType.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestType.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java
similarity index 100%
rename from lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java
diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java
index e8599f4b985c1..53151db4a0a74 100644
--- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java
+++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java
@@ -20,7 +20,7 @@
 
 import org.apache.iotdb.commons.utils.FileUtils;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
 import org.apache.iotdb.tsfile.utils.Pair;
 
 import org.junit.After;

From 11d05a2a4162a89b4b67fda104c683f6b2891e24 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sat, 15 Oct 2022 11:43:17 +0800
Subject: [PATCH 32/53] Modify the abstract class WalRecord to an interface

---
 .../tagSchemaRegion/tagIndex/wal/WALEntry.java   | 16 +++++++++++++++-
 .../java/org/apache/iotdb/lsm/wal/WALRecord.java | 16 +++++-----------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java
index 5d73ed74ae60c..c2874a8290daf 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java
@@ -28,7 +28,7 @@
 import java.util.List;
 
 /** represents a record in the wal file */
-public class WALEntry extends WALRecord {
+public class WALEntry implements WALRecord {
 
   // can be insertion(1) or deletion(2)
   private int type;
@@ -83,6 +83,20 @@ public void deserialize(DataInputStream stream) throws IOException {
     }
   }
 
+  /**
+   * generate wal record using prototyping pattern
+   *
+   * @return wal record
+   */
+  @Override
+  public WALRecord clone() {
+    try {
+      return (WALRecord) super.clone();
+    } catch (CloneNotSupportedException e) {
+      throw new AssertionError(e.getMessage());
+    }
+  }
+
   public int getType() {
     return type;
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
index 0aaf77460aab1..32175edc9c9bf 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
@@ -23,14 +23,14 @@
 import java.nio.ByteBuffer;
 
 /** represents a wal record, which can be extended to implement more complex wal records */
-public abstract class WALRecord implements Cloneable {
+public interface WALRecord extends Cloneable {
 
   /**
    * serialize the wal record
    *
    * @param buffer byte buffer
    */
-  public abstract void serialize(ByteBuffer buffer);
+  void serialize(ByteBuffer buffer);
 
   /**
    * deserialize via input stream
@@ -38,14 +38,8 @@ public abstract class WALRecord implements Cloneable {
    * @param stream data input stream
    * @throws IOException
    */
-  public abstract void deserialize(DataInputStream stream) throws IOException;
+  void deserialize(DataInputStream stream) throws IOException;
 
-  @Override
-  public WALRecord clone() {
-    try {
-      return (WALRecord) super.clone();
-    } catch (CloneNotSupportedException e) {
-      throw new AssertionError(e.getMessage());
-    }
-  }
+  // generate wal record using prototyping pattern
+  WALRecord clone();
 }

From 24a041accf9e20726d8f9acd391461358b7d21f9 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sat, 15 Oct 2022 17:15:08 +0800
Subject: [PATCH 33/53] IDTableWithDeviceIDListImpl

---
 schema-engine-tag/pom.xml                     |   5 -
 .../src/assembly/schema-engine-tag.xml        |   1 -
 .../tagSchemaRegion/TagSchemaRegion.java      |  84 ++++---
 .../AppendOnlyDeviceIDListFileManager.java    | 116 ----------
 .../deviceidlist/DeviceIDList.java            |  72 ------
 .../deviceidlist/IDeviceIDList.java           |  61 -----
 .../idtable/IDTableWithDeviceIDListImpl.java  | 124 ++++++++++
 .../deviceidlist/DeviceIDListTest.java        | 126 -----------
 .../IDTableWithDeviceIDListImplTest.java      | 212 ++++++++++++++++++
 9 files changed, 377 insertions(+), 424 deletions(-)
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java
 delete mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java
 create mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImplTest.java

diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml
index 184921f098956..04ea391c9285a 100644
--- a/schema-engine-tag/pom.xml
+++ b/schema-engine-tag/pom.xml
@@ -34,11 +34,6 @@
             RoaringBitmap
             0.9.32
         
-        
-            org.apache.iotdb
-            iotdb-lsm
-            ${project.version}
-        
         
             org.apache.iotdb
             iotdb-server
diff --git a/schema-engine-tag/src/assembly/schema-engine-tag.xml b/schema-engine-tag/src/assembly/schema-engine-tag.xml
index 49647f46a7a14..fa8f63ef5da1b 100644
--- a/schema-engine-tag/src/assembly/schema-engine-tag.xml
+++ b/schema-engine-tag/src/assembly/schema-engine-tag.xml
@@ -30,7 +30,6 @@
         
             /lib/tag-schema-region
             
-                org.apache.iotdb:iotdb-lsm
                 org.roaringbitmap:RoaringBitmap
                 org.apache.iotdb:schema-engine-tag
             
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
index be7a58511bc36..8a01a75fe2a68 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
@@ -34,10 +34,7 @@
 import org.apache.iotdb.db.exception.metadata.PathNotExistException;
 import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException;
 import org.apache.iotdb.db.metadata.LocalSchemaProcessor;
-import org.apache.iotdb.db.metadata.idtable.IDTable;
-import org.apache.iotdb.db.metadata.idtable.IDTableHashmapImpl;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry;
-import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
 import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry;
 import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
 import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode;
@@ -50,8 +47,7 @@
 import org.apache.iotdb.db.metadata.path.MeasurementPath;
 import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion;
 import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist.DeviceIDList;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist.IDeviceIDList;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.idtable.IDTableWithDeviceIDListImpl;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.MeasurementPathUtils;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.PathTagConverterUtils;
@@ -114,11 +110,8 @@ public class TagSchemaRegion implements ISchemaRegion {
   // tag inverted index
   private final TagInvertedIndex tagInvertedIndex;
 
-  // manager device id -> INT32 id
-  private final IDeviceIDList deviceIDList;
-
-  // manager timeSeries
-  private final IDTable idTable;
+  // manager timeSeries, and use a deviceID list manager device id -> INT32 id
+  private final IDTableWithDeviceIDListImpl idTableWithDeviceIDList;
 
   private final ISeriesNumerLimiter seriesNumerLimiter;
 
@@ -135,9 +128,8 @@ public TagSchemaRegion(
     this.storageGroupMNode = storageGroupMNode;
     this.seriesNumerLimiter = seriesNumerLimiter;
     File schemaRegionDir = new File(schemaRegionDirPath);
-    idTable = new IDTableHashmapImpl(schemaRegionDir);
+    idTableWithDeviceIDList = new IDTableWithDeviceIDListImpl(schemaRegionDir);
     tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath);
-    deviceIDList = new DeviceIDList(schemaRegionDirPath);
     init();
   }
 
@@ -168,7 +160,7 @@ public void init() throws MetadataException {
   public void clear() {
     try {
       tagInvertedIndex.clear();
-      deviceIDList.clear();
+      idTableWithDeviceIDList.clear();
     } catch (IOException e) {
       logger.error("clear tag inverted index failed", e);
     }
@@ -208,16 +200,14 @@ public void loadSnapshot(File latestSnapshotRootDir) {
   }
 
   private void createTagInvertedIndex(PartialPath devicePath) {
-    IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
     Map tagsMap =
         PathTagConverterUtils.pathToTags(storageGroupFullPath, devicePath.getFullPath());
-    synchronized (deviceIDList) {
-      deviceIDList.add(deviceID);
-      tagInvertedIndex.addTags(tagsMap, deviceIDList.size() - 1);
+    synchronized (idTableWithDeviceIDList) {
+      tagInvertedIndex.addTags(tagsMap, idTableWithDeviceIDList.size() - 1);
     }
   }
 
-  private List getDeviceIDsByInvertedIndex(PartialPath path) {
+  private List getDeviceIDsFromInvertedIndex(PartialPath path) {
     Map tags =
         PathTagConverterUtils.pathToTags(storageGroupFullPath, path.getFullPath());
     return tagInvertedIndex.getMatchedIDs(tags);
@@ -256,7 +246,7 @@ public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws Meta
                 + plan.getPath().getMeasurement());
     plan.setPath(path);
     devicePath = plan.getPath().getDevicePath();
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath());
+    DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath.getFullPath());
     if (deviceEntry != null) {
       if (deviceEntry.isAligned()) {
         throw new AlignedTimeseriesException(
@@ -267,7 +257,7 @@ public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws Meta
             devicePath.getFullPath() + "." + plan.getPath().getMeasurement());
       }
     }
-    idTable.createTimeseries(plan);
+    idTableWithDeviceIDList.createTimeseries(plan);
     // write the device path for the first time
     if (deviceEntry == null) {
       createTagInvertedIndex(devicePath);
@@ -283,7 +273,7 @@ public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws Met
                 storageGroupFullPath, devicePath.getFullPath()));
     plan.setPrefixPath(path);
     devicePath = plan.getPrefixPath();
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath());
+    DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath.getFullPath());
     if (deviceEntry != null) {
       if (!deviceEntry.isAligned()) {
         throw new AlignedTimeseriesException(
@@ -295,7 +285,7 @@ public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws Met
           throw new PathAlreadyExistException(devicePath.getFullPath());
       }
     }
-    idTable.createAlignedTimeseries(plan);
+    idTableWithDeviceIDList.createAlignedTimeseries(plan);
     // write the device path for the first time
     if (deviceEntry == null) {
       createTagInvertedIndex(devicePath);
@@ -371,7 +361,12 @@ public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch)
     int res = 0;
     List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern);
     for (IDeviceID deviceID : deviceIDs) {
-      res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size();
+      res +=
+          idTableWithDeviceIDList
+              .getDeviceEntry(deviceID.toStringID())
+              .getMeasurementMap()
+              .keySet()
+              .size();
     }
     return res;
   }
@@ -411,11 +406,11 @@ public Map getMeasurementCountGroupByLevel(
   @Override
   public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch)
       throws MetadataException {
-    synchronized (deviceIDList) {
+    synchronized (idTableWithDeviceIDList) {
       if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) {
-        return deviceIDList.size();
+        return idTableWithDeviceIDList.size();
       } else {
-        return getDeviceIDsByInvertedIndex(pathPattern).size();
+        return getDeviceIDsFromInvertedIndex(pathPattern).size();
       }
     }
   }
@@ -460,7 +455,7 @@ public Set getMatchedDevices(PartialPath pathPattern, boolean isPre
     String devicePath = pathPattern.getFullPath();
     // exact query
     if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) {
-      DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath);
+      DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath);
       if (deviceEntry != null) {
         matchedDevices.add(pathPattern);
       }
@@ -478,7 +473,7 @@ private List getDevicePaths(List deviceIDS) {
     if (config.getDeviceIDTransformationMethod().equals("SHA256")) {
       List schemaEntries = new ArrayList<>();
       for (IDeviceID deviceID : deviceIDS) {
-        DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID());
+        DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(deviceID.toStringID());
         Map map = deviceEntry.getMeasurementMap();
         // For each device, only one SchemaEntry needs to be obtained
         for (Map.Entry entry : map.entrySet()) {
@@ -486,7 +481,8 @@ private List getDevicePaths(List deviceIDS) {
           break;
         }
       }
-      List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries);
+      List diskSchemaEntries =
+          idTableWithDeviceIDList.getDiskSchemaEntries(schemaEntries);
       for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) {
         devicePaths.add(diskSchemaEntry.getDevicePath());
       }
@@ -501,7 +497,7 @@ private List getDevicePaths(List deviceIDS) {
   private List getSchemaEntries(List deviceIDS) {
     List schemaEntries = new ArrayList<>();
     for (IDeviceID deviceID : deviceIDS) {
-      DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID());
+      DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(deviceID.toStringID());
       Map schemaMap = deviceEntry.getMeasurementMap();
       for (Map.Entry entry : schemaMap.entrySet()) {
         schemaEntries.add(entry.getValue());
@@ -515,7 +511,8 @@ private List getMeasurementPaths(List deviceIDS)
     List measurementPaths = new ArrayList<>();
     if (config.getDeviceIDTransformationMethod().equals("SHA256")) {
       List schemaEntries = getSchemaEntries(deviceIDS);
-      List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries);
+      List diskSchemaEntries =
+          idTableWithDeviceIDList.getDiskSchemaEntries(schemaEntries);
       for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) {
         MeasurementPath measurementPath =
             MeasurementPathUtils.generateMeasurementPath(diskSchemaEntry);
@@ -523,7 +520,7 @@ private List getMeasurementPaths(List deviceIDS)
       }
     } else {
       for (IDeviceID deviceID : deviceIDS) {
-        DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID());
+        DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(deviceID.toStringID());
         Map schemaMap = deviceEntry.getMeasurementMap();
         for (Map.Entry entry : schemaMap.entrySet()) {
           MeasurementPath measurementPath =
@@ -558,7 +555,7 @@ private List getMeasurementPathsWithPointQuery(
     List measurementPaths = new LinkedList<>();
     String path =
         PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, devicePath.getFullPath());
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(path);
+    DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(path);
     if (deviceEntry == null) return measurementPaths;
     Map schemaMap = deviceEntry.getMeasurementMap();
     for (Map.Entry entry : schemaMap.entrySet()) {
@@ -600,7 +597,7 @@ public Pair, Integer> showTimeseries(
     // point query
     if (!path.endsWith(TAIL)) {
       path = PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, path);
-      DeviceEntry deviceEntry = idTable.getDeviceEntry(path);
+      DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(path);
       if (deviceEntry != null) {
         Map measurementMap = deviceEntry.getMeasurementMap();
         for (String m : measurementMap.keySet()) {
@@ -623,13 +620,14 @@ public Pair, Integer> showTimeseries(
   private void getTimeSeriesResultOfDeviceFromIDTable(
       List ShowTimeSeriesResults, IDeviceID deviceID) {
     Map measurementMap =
-        idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap();
+        idTableWithDeviceIDList.getDeviceEntry(deviceID.toStringID()).getMeasurementMap();
     if (deviceID instanceof SHA256DeviceID) {
       for (String m : measurementMap.keySet()) {
         SchemaEntry schemaEntry = measurementMap.get(m);
         List schemaEntries = new ArrayList<>();
         schemaEntries.add(schemaEntry);
-        List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries);
+        List diskSchemaEntries =
+            idTableWithDeviceIDList.getDiskSchemaEntries(schemaEntries);
         DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0);
         ShowTimeSeriesResults.add(
             ShowTimeSeriesResultUtils.generateShowTimeSeriesResult(
@@ -652,15 +650,15 @@ private List getDeviceIdFromInvertedIndex(PartialPath devicePath)
       path = path.substring(0, path.length() - TAIL.length());
       devicePath = new PartialPath(path);
     }
-    synchronized (deviceIDList) {
+    synchronized (idTableWithDeviceIDList) {
       if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) {
-        return deviceIDList.getAllDeviceIDS();
+        return idTableWithDeviceIDList.getAllDeviceIDS();
       } else {
         List IDS = new LinkedList<>();
-        List ids = getDeviceIDsByInvertedIndex(devicePath);
+        List ids = getDeviceIDsFromInvertedIndex(devicePath);
         if (ids.size() > 0) {
           for (int id : ids) {
-            IDS.add(deviceIDList.get(id));
+            IDS.add(idTableWithDeviceIDList.get(id));
           }
         }
         return IDS;
@@ -676,7 +674,7 @@ public List getAllMeasurementByDevicePath(PartialPath devicePat
 
   @Override
   public IMNode getDeviceNode(PartialPath path) throws MetadataException {
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath());
+    DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(path.getFullPath());
     if (deviceEntry == null) throw new PathNotExistException(path.getFullPath());
     return new EntityMNode(storageGroupMNode, path.getFullPath());
   }
@@ -745,7 +743,7 @@ public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan)
     checkAlignedAndAutoCreateSeries(plan);
     IMNode deviceMNode = getDeviceNode(devicePath);
     IMeasurementMNode measurementMNode;
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath());
+    DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath.getFullPath());
     Map schemaMap = deviceEntry.getMeasurementMap();
     for (int i = 0; i < measurementList.length; i++) {
       SchemaEntry schemaEntry = schemaMap.get(measurementList[i]);
@@ -820,7 +818,7 @@ public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate(
   }
 
   private SchemaEntry getSchemaEntry(String devicePath, String measurementName) {
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath);
+    DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath);
     if (deviceEntry == null) return null;
     return deviceEntry.getSchemaEntry(measurementName);
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java
deleted file mode 100644
index 6dda97a00aa1b..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist;
-
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-/** for append-only writing of device id list to disk */
-public class AppendOnlyDeviceIDListFileManager {
-
-  private static final Logger logger =
-      LoggerFactory.getLogger(AppendOnlyDeviceIDListFileManager.class);
-
-  private static final String DEVICE_ID_LIST_FILE_NAME = "device_id.list";
-
-  private boolean isRecover;
-
-  private String schemaDirPath;
-
-  private File deviceIDSFile;
-
-  private FileOutputStream outputStream;
-
-  public AppendOnlyDeviceIDListFileManager(String schemaDirPath) {
-    this.schemaDirPath = schemaDirPath;
-    isRecover = true;
-    try {
-      initFile();
-      outputStream = new FileOutputStream(deviceIDSFile, true);
-    } catch (IOException e) {
-      logger.error(e.getMessage());
-      throw new IllegalArgumentException(
-          "can't initialize device id list file at " + deviceIDSFile);
-    }
-  }
-
-  private void initFile() throws IOException {
-    File schemaDir = new File(schemaDirPath);
-    schemaDir.mkdirs();
-    deviceIDSFile = new File(schemaDir, DEVICE_ID_LIST_FILE_NAME);
-    if (!deviceIDSFile.exists()) {
-      // create new file
-      deviceIDSFile.createNewFile();
-    }
-  }
-
-  /**
-   * write the device id to file
-   *
-   * @param deviceID device id
-   */
-  public void write(String deviceID) {
-    try {
-      if (!isRecover) {
-        ReadWriteIOUtils.write(deviceID, outputStream);
-      }
-    } catch (IOException e) {
-      logger.error("failed to write device id: " + deviceID);
-      throw new IllegalArgumentException("can't write device id of " + deviceID);
-    }
-  }
-
-  /**
-   * recover device id list
-   *
-   * @param deviceIDList device id list
-   */
-  public void recover(DeviceIDList deviceIDList) {
-    logger.info("recover device id list using file {}", deviceIDSFile);
-    try (FileInputStream inputStream = new FileInputStream(deviceIDSFile)) {
-      while (inputStream.available() > 0) {
-        String deviceID = ReadWriteIOUtils.readString(inputStream);
-        deviceIDList.add(DeviceIDFactory.getInstance().getDeviceID(deviceID));
-      }
-    } catch (IOException e) {
-      logger.info("device id list is incomplete, we will recover as much as we can.");
-    }
-    isRecover = false;
-  }
-
-  @TestOnly
-  public void close() throws IOException {
-    try {
-      outputStream.close();
-      outputStream = null;
-    } catch (IOException e) {
-      logger.error("close device id list file failed");
-      throw e;
-    }
-  }
-}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java
deleted file mode 100644
index 4099248724d71..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist;
-
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/** manage device id -> int32 id */
-public class DeviceIDList implements IDeviceIDList {
-
-  // use an array list to manage device id -> int id
-  private final List deviceIDS;
-
-  private AppendOnlyDeviceIDListFileManager appendOnlyDeviceIDListFileManager;
-
-  public DeviceIDList(String schemaDirPath) {
-    deviceIDS = new ArrayList<>();
-    appendOnlyDeviceIDListFileManager = new AppendOnlyDeviceIDListFileManager(schemaDirPath);
-    recover();
-  }
-
-  public void recover() {
-    appendOnlyDeviceIDListFileManager.recover(this);
-  }
-
-  @Override
-  public void add(IDeviceID deviceID) {
-    deviceIDS.add(deviceID);
-    appendOnlyDeviceIDListFileManager.write(deviceID.toStringID());
-  }
-
-  @Override
-  public IDeviceID get(int index) {
-    return deviceIDS.get(index);
-  }
-
-  @Override
-  public int size() {
-    return deviceIDS.size();
-  }
-
-  @Override
-  public List getAllDeviceIDS() {
-    return new ArrayList<>(deviceIDS);
-  }
-
-  @TestOnly
-  public void clear() throws IOException {
-    appendOnlyDeviceIDListFileManager.close();
-    deviceIDS.clear();
-  }
-}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java
deleted file mode 100644
index 3d461a4b7097a..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist;
-
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
-
-import java.io.IOException;
-import java.util.List;
-
-/** manage device id -> int32 id */
-public interface IDeviceIDList {
-
-  /**
-   * insert a device id
-   *
-   * @param deviceID device id
-   */
-  void add(IDeviceID deviceID);
-
-  /**
-   * get device id using int32 id
-   *
-   * @param index int32 id
-   * @return device id
-   */
-  IDeviceID get(int index);
-
-  /**
-   * returns the number of managed device ids
-   *
-   * @return the number of managed device ids
-   */
-  int size();
-
-  /**
-   * get all managed device ids
-   *
-   * @return all managed device ids
-   */
-  List getAllDeviceIDS();
-
-  @TestOnly
-  void clear() throws IOException;
-}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java
new file mode 100644
index 0000000000000..94565125a2136
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.idtable;
+
+import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.db.metadata.idtable.IDTableHashmapImpl;
+import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry;
+import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
+import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
+import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry;
+import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class IDTableWithDeviceIDListImpl extends IDTableHashmapImpl {
+
+  private List deviceIDS;
+
+  public IDTableWithDeviceIDListImpl(File storageGroupDir) {
+    super(storageGroupDir);
+    if (deviceIDS == null) {
+      deviceIDS = new ArrayList<>();
+    }
+  }
+
+  /**
+   * Whether device entry already exists
+   *
+   * @param devicePath device path
+   * @return false if the device entry already exists, otherwise return true
+   */
+  private boolean deviceEntryNotExist(String devicePath) {
+    DeviceEntry deviceEntry = getDeviceEntry(devicePath);
+    return deviceEntry == null;
+  }
+  /**
+   * create aligned timeseries
+   *
+   * @param plan create aligned timeseries plan
+   * @throws MetadataException
+   */
+  @Override
+  public synchronized void createAlignedTimeseries(CreateAlignedTimeSeriesPlan plan)
+      throws MetadataException {
+    String devicePath = plan.getPrefixPath().getFullPath();
+    if (deviceEntryNotExist(devicePath)) {
+      IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
+      deviceIDS.add(deviceID);
+    }
+    super.createAlignedTimeseries(plan);
+  }
+
+  @Override
+  public synchronized void createTimeseries(CreateTimeSeriesPlan plan) throws MetadataException {
+    String devicePath = plan.getPath().getDevicePath().getFullPath();
+    if (deviceEntryNotExist(devicePath)) {
+      IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
+      deviceIDS.add(deviceID);
+    }
+    super.createTimeseries(plan);
+  }
+
+  /**
+   * put schema entry to id table, currently used in recover
+   *
+   * @param devicePath device path (can be device id formed path)
+   * @param measurement measurement name
+   * @param schemaEntry schema entry to put
+   * @param isAligned is the device aligned
+   */
+  @Override
+  public void putSchemaEntry(
+      String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned)
+      throws MetadataException {
+    if (deviceIDS == null) {
+      deviceIDS = new ArrayList<>();
+    }
+    if (deviceEntryNotExist(devicePath)) {
+      IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
+      deviceIDS.add(deviceID);
+    }
+    super.putSchemaEntry(devicePath, measurement, schemaEntry, isAligned);
+  }
+
+  @Override
+  @TestOnly
+  public void clear() throws IOException {
+    super.clear();
+    deviceIDS = null;
+  }
+
+  public synchronized IDeviceID get(int index) {
+    return deviceIDS.get(index);
+  }
+
+  public int size() {
+    return deviceIDS.size();
+  }
+
+  public List getAllDeviceIDS() {
+    return new ArrayList<>(deviceIDS);
+  }
+}
diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java
deleted file mode 100644
index 26daa70890f1d..0000000000000
--- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist;
-
-import org.apache.iotdb.commons.utils.FileUtils;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-public class DeviceIDListTest {
-
-  private IDeviceIDList deviceIDList;
-
-  private String storageGroupDirPath;
-
-  private String schemaRegionDirPath;
-
-  private String storageGroupFullPath = "root/testDeviceIDList";
-
-  private boolean isEnableIDTable;
-
-  private String originalDeviceIDTransformationMethod;
-
-  private String schemaDir;
-
-  private String[] devicePaths =
-      new String[] {
-        storageGroupFullPath + ".a.b.c.d1",
-        storageGroupFullPath + ".a.b.c.d2",
-        storageGroupFullPath + ".a.b.c.d3",
-        storageGroupFullPath + ".a.b.c.d4",
-        storageGroupFullPath + ".a.b.d.d1",
-        storageGroupFullPath + ".a.b.d.d2",
-        storageGroupFullPath + ".a.b.e.d1",
-        storageGroupFullPath + ".a.b.f.d1",
-        storageGroupFullPath + ".a.b.g.d1",
-        storageGroupFullPath + ".a.b.h.d1",
-      };
-
-  @Before
-  public void setUp() throws Exception {
-    schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
-    isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable();
-    originalDeviceIDTransformationMethod =
-        IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod();
-    IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true);
-    IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256");
-    storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath;
-    schemaRegionDirPath = storageGroupDirPath + File.separator + 0;
-    deviceIDList = new DeviceIDList(schemaRegionDirPath);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    deviceIDList.clear();
-    IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable);
-    IoTDBDescriptor.getInstance()
-        .getConfig()
-        .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod);
-    FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir));
-  }
-
-  @Test
-  public void testAddandGetDeviceID() {
-    List deviceIDS = generateTestDeviceIDS();
-    for (IDeviceID deviceID : deviceIDS) {
-      deviceIDList.add(deviceID);
-      System.out.println(deviceID.toStringID());
-    }
-    for (int i = 0; i < deviceIDS.size(); i++) {
-      assertEquals(deviceIDS.get(i), deviceIDList.get(i));
-    }
-  }
-
-  @Test
-  public void testRecover() throws IOException {
-    List deviceIDS = generateTestDeviceIDS();
-    for (IDeviceID deviceID : deviceIDS) {
-      deviceIDList.add(deviceID);
-      System.out.println(deviceID.toStringID());
-    }
-
-    deviceIDList.clear();
-
-    deviceIDList = new DeviceIDList(schemaRegionDirPath);
-    for (int i = 0; i < deviceIDS.size(); i++) {
-      assertEquals(deviceIDS.get(i), deviceIDList.get(i));
-    }
-  }
-
-  private List generateTestDeviceIDS() {
-    List deviceIDS = new ArrayList<>();
-    for (String devicePath : devicePaths) {
-      deviceIDS.add(DeviceIDFactory.getInstance().getDeviceID(devicePath));
-    }
-    return deviceIDS;
-  }
-}
diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImplTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImplTest.java
new file mode 100644
index 0000000000000..2e9b4ccde8051
--- /dev/null
+++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImplTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.idtable;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class IDTableWithDeviceIDListImplTest {
+
+  private CompressionType compressionType;
+
+  private String storageGroupDirPath;
+
+  private String schemaRegionDirPath;
+
+  private String storageGroupFullPath = "root/testIDTableWithDeviceIDListImpl";
+
+  private String storageGroup = "root.testIDTableWithDeviceIDListImpl";
+
+  private boolean isEnableIDTable = false;
+
+  private String originalDeviceIDTransformationMethod = null;
+
+  private boolean isEnableIDTableLogFile = false;
+
+  private String schemaDir;
+
+  private IDTableWithDeviceIDListImpl idTableWithDeviceIDList;
+
+  @Before
+  public void before() {
+    compressionType = TSFileDescriptor.getInstance().getConfig().getCompressor();
+    schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
+    isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable();
+    originalDeviceIDTransformationMethod =
+        IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod();
+    isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile();
+    IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true);
+    IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256");
+    IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true);
+    storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath;
+    schemaRegionDirPath = storageGroupDirPath + File.separator + 0;
+    idTableWithDeviceIDList = new IDTableWithDeviceIDListImpl(new File(schemaRegionDirPath));
+  }
+
+  @After
+  public void clean() throws IOException, StorageEngineException {
+    idTableWithDeviceIDList.clear();
+    IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable);
+    IoTDBDescriptor.getInstance()
+        .getConfig()
+        .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod);
+    IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile);
+    FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir));
+  }
+
+  @Test
+  public void testCreateAlignedTimeseries() {
+    try {
+      createAlignedTimeseries();
+      assertEquals(idTableWithDeviceIDList.size(), 2);
+      assertEquals(
+          idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d1.aligned_device").getDeviceID(),
+          idTableWithDeviceIDList.get(0));
+      assertEquals(
+          idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d2.aligned_device").getDeviceID(),
+          idTableWithDeviceIDList.get(1));
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testCreateTimeseries() {
+    try {
+      createTimeseries();
+      assertEquals(idTableWithDeviceIDList.size(), 2);
+      assertEquals(
+          idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d1").getDeviceID(),
+          idTableWithDeviceIDList.get(0));
+      assertEquals(
+          idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d2").getDeviceID(),
+          idTableWithDeviceIDList.get(1));
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testRecover() {
+    try {
+      createTimeseries();
+      createAlignedTimeseries();
+
+      idTableWithDeviceIDList.clear();
+      idTableWithDeviceIDList = new IDTableWithDeviceIDListImpl(new File(schemaRegionDirPath));
+
+      assertEquals(idTableWithDeviceIDList.size(), 4);
+      assertEquals(
+          idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d1").getDeviceID(),
+          idTableWithDeviceIDList.get(0));
+      assertEquals(
+          idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d2").getDeviceID(),
+          idTableWithDeviceIDList.get(1));
+      assertEquals(
+          idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d1.aligned_device").getDeviceID(),
+          idTableWithDeviceIDList.get(2));
+      assertEquals(
+          idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d2.aligned_device").getDeviceID(),
+          idTableWithDeviceIDList.get(3));
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  private void createAlignedTimeseries() throws Exception {
+    CreateAlignedTimeSeriesPlan plan =
+        new CreateAlignedTimeSeriesPlan(
+            new PartialPath(storageGroup + ".d1.aligned_device"),
+            Arrays.asList("s1", "s2", "s3"),
+            Arrays.asList(
+                TSDataType.valueOf("FLOAT"),
+                TSDataType.valueOf("INT64"),
+                TSDataType.valueOf("INT32")),
+            Arrays.asList(
+                TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")),
+            Arrays.asList(compressionType, compressionType, compressionType),
+            null,
+            null,
+            null);
+    idTableWithDeviceIDList.createAlignedTimeseries(plan);
+    plan =
+        new CreateAlignedTimeSeriesPlan(
+            new PartialPath(storageGroup + ".d2.aligned_device"),
+            Arrays.asList("s1", "s2", "s3"),
+            Arrays.asList(
+                TSDataType.valueOf("FLOAT"),
+                TSDataType.valueOf("INT64"),
+                TSDataType.valueOf("INT32")),
+            Arrays.asList(
+                TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")),
+            Arrays.asList(compressionType, compressionType, compressionType),
+            null,
+            null,
+            null);
+    idTableWithDeviceIDList.createAlignedTimeseries(plan);
+  }
+
+  private void createTimeseries() throws Exception {
+    CreateTimeSeriesPlan createTimeSeriesPlan =
+        new CreateTimeSeriesPlan(
+            new PartialPath(storageGroup + ".d1.s0"),
+            TSDataType.valueOf("INT32"),
+            TSEncoding.valueOf("RLE"),
+            compressionType,
+            Collections.emptyMap(),
+            null,
+            null,
+            null);
+    idTableWithDeviceIDList.createTimeseries(createTimeSeriesPlan);
+    createTimeSeriesPlan =
+        new CreateTimeSeriesPlan(
+            new PartialPath(storageGroup + ".d2.s0"),
+            TSDataType.valueOf("INT32"),
+            TSEncoding.valueOf("RLE"),
+            compressionType,
+            Collections.emptyMap(),
+            null,
+            null,
+            null);
+    idTableWithDeviceIDList.createTimeseries(createTimeSeriesPlan);
+  }
+}

From 23ec097b20dcd1dcc0d77f8f5d50cd046de834b9 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sat, 15 Oct 2022 22:57:09 +0800
Subject: [PATCH 34/53] add LevelProcessChain and TagSchemaRegionTest

---
 .../idtable/IDTableWithDeviceIDListImpl.java  |  14 ++
 .../tagIndex/TagInvertedIndex.java            |   9 +-
 .../tagIndex/deletion/DeletionManager.java    |  10 +-
 .../tagIndex/insertion/InsertionManager.java  |  10 +-
 .../tagIndex/query/MemChunkGroupQuery.java    |   2 +-
 .../tagIndex/query/QueryManager.java          |  10 +-
 .../lsm/levelProcess/LevelProcessChain.java   |  36 +++
 ...icLsmManager.java => BasicLSMManager.java} |  28 +--
 .../{LsmManager.java => LSMManager.java}      |  27 +-
 .../tagSchemaRegion/TagSchemaRegionTest.java  | 231 ++++++++++++++++++
 .../db/engine/storagegroup/DataRegion.java    |   2 +-
 11 files changed, 342 insertions(+), 37 deletions(-)
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/{BasicLsmManager.java => BasicLSMManager.java} (75%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/{LsmManager.java => LSMManager.java} (71%)
 create mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java
index 94565125a2136..4f506d5461226 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java
@@ -33,8 +33,13 @@
 import java.util.ArrayList;
 import java.util.List;
 
+/**
+ * an idTable implementation，manager timeSeries, and use a deviceID list manager device id -> INT32
+ * id
+ */
 public class IDTableWithDeviceIDListImpl extends IDTableHashmapImpl {
 
+  // use a deviceID list manager device id -> INT32 id
   private List deviceIDS;
 
   public IDTableWithDeviceIDListImpl(File storageGroupDir) {
@@ -64,6 +69,7 @@ private boolean deviceEntryNotExist(String devicePath) {
   public synchronized void createAlignedTimeseries(CreateAlignedTimeSeriesPlan plan)
       throws MetadataException {
     String devicePath = plan.getPrefixPath().getFullPath();
+    // This device path is created for the first time, append it to deviceIdList
     if (deviceEntryNotExist(devicePath)) {
       IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
       deviceIDS.add(deviceID);
@@ -71,9 +77,16 @@ public synchronized void createAlignedTimeseries(CreateAlignedTimeSeriesPlan pla
     super.createAlignedTimeseries(plan);
   }
 
+  /**
+   * create timeseries
+   *
+   * @param plan create timeseries plan
+   * @throws MetadataException
+   */
   @Override
   public synchronized void createTimeseries(CreateTimeSeriesPlan plan) throws MetadataException {
     String devicePath = plan.getPath().getDevicePath().getFullPath();
+    // This device path is created for the first time, append it to deviceIdList
     if (deviceEntryNotExist(devicePath)) {
       IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
       deviceIDS.add(deviceID);
@@ -96,6 +109,7 @@ public void putSchemaEntry(
     if (deviceIDS == null) {
       deviceIDS = new ArrayList<>();
     }
+    // This device path is created for the first time, append it to deviceIdList
     if (deviceEntryNotExist(devicePath)) {
       IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
       deviceIDS.add(deviceID);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index a40decbf786c4..b4b89baf44160 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -201,9 +201,12 @@ public synchronized List getMatchedIDs(Map tags) {
     try {
       for (Map.Entry tag : tags.entrySet()) {
         RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue());
-        if (i == 0) roaringBitmap = rb;
-        else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb);
-        i++;
+        if (rb == null) continue;
+        else {
+          if (i == 0) roaringBitmap = rb;
+          else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb);
+          i++;
+        }
       }
     } catch (Exception e) {
       logger.error(e.getMessage());
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
index 51fd8ab031a22..ff2911ed9cfe2 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
@@ -21,10 +21,11 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.manager.BasicLsmManager;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.manager.BasicLSMManager;
 
 /** manage deletion to MemTable */
-public class DeletionManager extends BasicLsmManager {
+public class DeletionManager extends BasicLSMManager {
 
   // use wal manager object to write wal file on deletion
   private WALManager walManager;
@@ -50,8 +51,11 @@ public void preProcess(MemTable root, DeleteRequestContext context) throws Excep
 
   /** set the delete operation for each layer of memory nodes */
   private void initLevelProcess() {
-    this.nextLevel(new MemTableDeletion())
+    LevelProcessChain levelProcessChain = new LevelProcessChain<>();
+    levelProcessChain
+        .nextLevel(new MemTableDeletion())
         .nextLevel(new MemChunkGroupDeletion())
         .nextLevel(new MemChunkDeletion());
+    setLevelProcessChain(levelProcessChain);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
index 583f8944fd147..3ad52edaaec47 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
@@ -21,12 +21,13 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.manager.BasicLsmManager;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.manager.BasicLSMManager;
 
 import java.io.IOException;
 
 /** manage insertion to MemTable */
-public class InsertionManager extends BasicLsmManager {
+public class InsertionManager extends BasicLSMManager {
 
   // use wal manager object to write wal file on insertion
   private WALManager walManager;
@@ -52,8 +53,11 @@ public void preProcess(MemTable root, InsertRequestContext context) throws IOExc
 
   /** set the insert operation for each layer of memory nodes */
   private void initLevelProcess() {
-    this.nextLevel(new MemTableInsertion())
+    LevelProcessChain levelProcessChain = new LevelProcessChain<>();
+    levelProcessChain
+        .nextLevel(new MemTableInsertion())
         .nextLevel(new MemChunkGroupInsertion())
         .nextLevel(new MemChunkInsertion());
+    setLevelProcessChain(levelProcessChain);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
index 1ba77a754b540..ea312468e028a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
@@ -46,7 +46,7 @@ public List getChildren(MemChunkGroup memNode, QueryRequestContext con
   }
 
   /**
-   * the insert method corresponding to the MemChunkGroup node
+   * the query method corresponding to the MemChunkGroup node
    *
    * @param memNode memory node
    * @param context query request context
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
index d3dc3a1d30538..38022663fa565 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
@@ -20,10 +20,11 @@
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.manager.BasicLsmManager;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.manager.BasicLSMManager;
 
 /** manage insertion to MemTable */
-public class QueryManager extends BasicLsmManager {
+public class QueryManager extends BasicLSMManager {
 
   public QueryManager() {
     initLevelProcess();
@@ -31,8 +32,11 @@ public QueryManager() {
 
   /** set the query operation for each layer of memory nodes */
   private void initLevelProcess() {
-    this.nextLevel(new MemTableQuery())
+    LevelProcessChain levelProcessChain = new LevelProcessChain<>();
+    levelProcessChain
+        .nextLevel(new MemTableQuery())
         .nextLevel(new MemChunkGroupQuery())
         .nextLevel(new MemChunkQuery());
+    setLevelProcessChain(levelProcessChain);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
new file mode 100644
index 0000000000000..deeaff99b4b3e
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.levelProcess;
+
+import org.apache.iotdb.lsm.context.RequestContext;
+
+public class LevelProcessChain {
+
+  // the level process of the first layer of memory nodes
+  LevelProcess headLevelProcess;
+
+  public  LevelProcess nextLevel(LevelProcess next) {
+    this.headLevelProcess = next;
+    return next;
+  }
+
+  public void process(T memNode, C context) {
+    headLevelProcess.process(memNode, context);
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
similarity index 75%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
index 5b8816f7f2f26..7740affd45ba6 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
@@ -19,13 +19,19 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.LevelProcess;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
 
 /** */
-public class BasicLsmManager implements LsmManager {
+public class BasicLSMManager implements LSMManager {
 
   // the level process of the first layer of memory nodes
-  LevelProcess levelProcess;
+  LevelProcessChain levelProcessChain;
+
+  public BasicLSMManager() {}
+
+  public BasicLSMManager(LevelProcessChain levelProcessChain) {
+    this.levelProcessChain = levelProcessChain;
+  }
 
   /**
    * preprocessing of the root memory node
@@ -34,6 +40,7 @@ public class BasicLsmManager implements LsmManager<
    * @param context request context
    * @throws Exception
    */
+  @Override
   public void preProcess(T root, C context) throws Exception {}
 
   /**
@@ -43,6 +50,7 @@ public void preProcess(T root, C context) throws Exception {}
    * @param context request context
    * @throws Exception
    */
+  @Override
   public void postProcess(T root, C context) throws Exception {}
 
   /**
@@ -55,19 +63,11 @@ public void postProcess(T root, C context) throws Exception {}
   @Override
   public void process(T root, C context) throws Exception {
     preProcess(root, context);
-    levelProcess.process(root, context);
+    levelProcessChain.process(root, context);
     postProcess(root, context);
   }
 
-  /**
-   * add the LevelProcess of the next layer of memory nodes
-   *
-   * @param levelProcess LevelProcess of the next layer
-   * @return LevelProcess of the next layer
-   */
-  @Override
-  public  LevelProcess nextLevel(LevelProcess levelProcess) {
-    this.levelProcess = levelProcess;
-    return levelProcess;
+  public void setLevelProcessChain(LevelProcessChain levelProcessChain) {
+    this.levelProcessChain = levelProcessChain;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java
similarity index 71%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java
index ea2355378e5d6..c5d4169a75dd2 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java
@@ -19,24 +19,33 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.LevelProcess;
 
 // used to implement lsm manager
-public interface LsmManager {
+public interface LSMManager {
 
   /**
-   * use this method to process root memory node
+   * preprocessing of the root memory node
    *
-   * @param memNode memory node
+   * @param root root memory node
    * @param context request context
+   * @throws Exception
    */
-  void process(T memNode, C context) throws Exception;
+  void preProcess(T root, C context) throws Exception;
+
+  /**
+   * postprocessing of the root memory node
+   *
+   * @param root root memory node
+   * @param context request context
+   * @throws Exception
+   */
+  void postProcess(T root, C context) throws Exception;
 
   /**
-   * add the LevelProcess of the next layer of memory nodes
+   * use this method to process root memory node
    *
-   * @param next LevelProcess of the next layer
-   * @return LevelProcess of the next layer
+   * @param memNode memory node
+   * @param context request context
    */
-   LevelProcess nextLevel(LevelProcess next);
+  void process(T memNode, C context) throws Exception;
 }
diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java
new file mode 100644
index 0000000000000..513d04783b1ff
--- /dev/null
+++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion;
+
+import org.apache.iotdb.commons.consensus.SchemaRegionId;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.metadata.path.MeasurementPath;
+import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class TagSchemaRegionTest {
+
+  private CompressionType compressionType;
+
+  private String storageGroupDirPath;
+
+  private String schemaRegionDirPath;
+
+  private String storageGroupFullPath = "root/testTagSchemaRegion";
+
+  private String storageGroup = "root.testTagSchemaRegion";
+
+  private boolean isEnableIDTable = false;
+
+  private String originalDeviceIDTransformationMethod = null;
+
+  private boolean isEnableIDTableLogFile = false;
+
+  private String schemaDir;
+
+  private TagSchemaRegion tagSchemaRegion;
+
+  @Before
+  public void before() {
+    compressionType = TSFileDescriptor.getInstance().getConfig().getCompressor();
+    schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
+    isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable();
+    originalDeviceIDTransformationMethod =
+        IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod();
+    isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile();
+    IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true);
+    IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256");
+    IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true);
+    storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath;
+    schemaRegionDirPath = storageGroupDirPath + File.separator + 0;
+  }
+
+  @After
+  public void clean() {
+    tagSchemaRegion.clear();
+    IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable);
+    IoTDBDescriptor.getInstance()
+        .getConfig()
+        .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod);
+    IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile);
+    FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir));
+  }
+
+  @Test
+  public void testCreateTimeseries() {
+    try {
+      tagSchemaRegion =
+          new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null);
+      createTimeseries();
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testCreateAlignedTimeSeries() {
+    try {
+      tagSchemaRegion =
+          new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null);
+      createAlignedTimeseries();
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testRecover() {
+    try {
+      tagSchemaRegion =
+          new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null);
+      createAlignedTimeseries();
+      createTimeseries();
+
+      tagSchemaRegion.clear();
+      tagSchemaRegion =
+          new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null);
+
+      getMeasurementPathsTest();
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testGetMeasurementPaths() {
+    try {
+      tagSchemaRegion =
+          new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null);
+      createAlignedTimeseries();
+      createTimeseries();
+      getMeasurementPathsTest();
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  private void getMeasurementPathsTest() throws Exception {
+    PartialPath pathPattern = new PartialPath(storageGroup + ".tag1.a.tag2.b.s0");
+    List measurementPaths =
+        tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+    assertEquals(measurementPaths.size(), 1);
+    assertEquals(measurementPaths.get(0).getFullPath(), storageGroup + ".tag1.a.tag2.b.s0");
+
+    pathPattern = new PartialPath(storageGroup + ".tag1.a.**.s");
+    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+    assertEquals(measurementPaths.size(), 5);
+
+    pathPattern = new PartialPath(storageGroup + ".tag2.b.**.s");
+    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+    assertEquals(measurementPaths.size(), 1);
+
+    pathPattern = new PartialPath(storageGroup + ".tag3.b.**.s");
+    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+    assertEquals(measurementPaths.size(), 3);
+
+    pathPattern = new PartialPath(storageGroup + ".tag2.y.tag1.x.**.s");
+    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+    assertEquals(measurementPaths.size(), 3);
+  }
+
+  private void createTimeseries() throws Exception {
+    CreateTimeSeriesPlan createTimeSeriesPlan =
+        new CreateTimeSeriesPlan(
+            new PartialPath(storageGroup + ".tag1.a.tag2.b.s0"),
+            TSDataType.valueOf("INT32"),
+            TSEncoding.valueOf("RLE"),
+            compressionType,
+            Collections.emptyMap(),
+            null,
+            null,
+            null);
+    tagSchemaRegion.createTimeseries(createTimeSeriesPlan, 0);
+    createTimeSeriesPlan =
+        new CreateTimeSeriesPlan(
+            new PartialPath(storageGroup + ".tag1.a.tag2.c.s0"),
+            TSDataType.valueOf("INT32"),
+            TSEncoding.valueOf("RLE"),
+            compressionType,
+            Collections.emptyMap(),
+            null,
+            null,
+            null);
+    tagSchemaRegion.createTimeseries(createTimeSeriesPlan, 0);
+  }
+
+  private void createAlignedTimeseries() throws Exception {
+    CreateAlignedTimeSeriesPlan plan =
+        new CreateAlignedTimeSeriesPlan(
+            new PartialPath(storageGroup + ".tag1.a.tag3.b"),
+            Arrays.asList("s1", "s2", "s3"),
+            Arrays.asList(
+                TSDataType.valueOf("FLOAT"),
+                TSDataType.valueOf("INT64"),
+                TSDataType.valueOf("INT32")),
+            Arrays.asList(
+                TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")),
+            Arrays.asList(compressionType, compressionType, compressionType),
+            null,
+            null,
+            null);
+    tagSchemaRegion.createAlignedTimeSeries(plan);
+    plan =
+        new CreateAlignedTimeSeriesPlan(
+            new PartialPath(storageGroup + "tag1.x.tag2.y"),
+            Arrays.asList("s1", "s2", "s3"),
+            Arrays.asList(
+                TSDataType.valueOf("FLOAT"),
+                TSDataType.valueOf("INT64"),
+                TSDataType.valueOf("INT32")),
+            Arrays.asList(
+                TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")),
+            Arrays.asList(compressionType, compressionType, compressionType),
+            null,
+            null,
+            null);
+    tagSchemaRegion.createAlignedTimeSeries(plan);
+  }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index bf8286c9a9fa6..79d84075c0861 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -303,7 +303,7 @@ public DataRegion(
     }
 
     // if use id table, we use id table flush time manager
-    if (config.isEnableIDTable()) {
+    if (config.isEnableIDTable() && !config.getSchemaEngineMode().equals("Tag")) {
       idTable = IDTableManager.getInstance().getIDTableDirectly(storageGroupName);
       lastFlushTimeManager = new IDTableFlushTimeManager(idTable);
     } else {

From c981031aba7ee066e3b15174c21280434eae6fdd Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sun, 16 Oct 2022 10:01:48 +0800
Subject: [PATCH 35/53] merge

---
 .../tagSchemaRegion/MockTagSchemaRegion.java  | 983 ------------------
 .../tagSchemaRegion/TagSchemaRegion.java      |  81 +-
 .../idtable/IDTableWithDeviceIDListImpl.java  |  10 +-
 .../utils/MeasurementPathUtils.java           |   2 +-
 .../tagSchemaRegion/TagSchemaRegionTest.java  |  12 +-
 .../common/schematree/ClusterSchemaTree.java  |   1 -
 6 files changed, 56 insertions(+), 1033 deletions(-)
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java
deleted file mode 100644
index d360a84327074..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java
+++ /dev/null
@@ -1,983 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion;
-
-import org.apache.iotdb.common.rpc.thrift.TSchemaNode;
-import org.apache.iotdb.commons.consensus.SchemaRegionId;
-import org.apache.iotdb.commons.exception.MetadataException;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.commons.path.PathPatternTree;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException;
-import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException;
-import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
-import org.apache.iotdb.db.exception.metadata.PathNotExistException;
-import org.apache.iotdb.db.metadata.LocalSchemaProcessor;
-import org.apache.iotdb.db.metadata.idtable.IDTable;
-import org.apache.iotdb.db.metadata.idtable.IDTableManager;
-import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry;
-import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
-import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
-import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode;
-import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID;
-import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry;
-import org.apache.iotdb.db.metadata.mnode.EntityMNode;
-import org.apache.iotdb.db.metadata.mnode.IMNode;
-import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
-import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode;
-import org.apache.iotdb.db.metadata.path.MeasurementPath;
-import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion;
-import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils;
-import org.apache.iotdb.db.metadata.template.Template;
-import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo;
-import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo;
-import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
-import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan;
-import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan;
-import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan;
-import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.dataset.ShowDevicesResult;
-import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult;
-import org.apache.iotdb.external.api.ISeriesNumerLimiter;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-
-import org.jetbrains.annotations.NotNull;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.Function;
-
-import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding;
-
-/** Mock tag schema region, only used for test */
-public class MockTagSchemaRegion implements ISchemaRegion {
-
-  protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
-
-  private final String TAIL = ".**";
-  private final IStorageGroupMNode storageGroupMNode;
-  private String storageGroupFullPath;
-  private SchemaRegionId schemaRegionId;
-
-  private Map>> tagInvertedIndex;
-
-  private List deviceIDS;
-
-  private IDTable idTable;
-
-  private final ISeriesNumerLimiter seriesNumerLimiter;
-
-  public MockTagSchemaRegion(
-      PartialPath storageGroup,
-      SchemaRegionId schemaRegionId,
-      IStorageGroupMNode storageGroupMNode,
-      ISeriesNumerLimiter seriesNumerLimiter)
-      throws MetadataException {
-
-    storageGroupFullPath = storageGroup.getFullPath();
-    this.schemaRegionId = schemaRegionId;
-    this.storageGroupMNode = storageGroupMNode;
-    this.deviceIDS = new ArrayList<>();
-    this.seriesNumerLimiter = seriesNumerLimiter;
-    tagInvertedIndex = new ConcurrentHashMap<>();
-    idTable = IDTableManager.getInstance().getIDTable(storageGroup);
-    init();
-  }
-
-  @NotNull
-  private Map pathToTags(String path) {
-    if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>();
-    String devicePath = path.substring(storageGroupFullPath.length() + 1);
-    String[] tags = devicePath.split("\\.");
-    Map tagsMap = new TreeMap<>();
-    for (int i = 0; i < tags.length; i += 2) {
-      tagsMap.put(tags[i], tags[i + 1]);
-    }
-    return tagsMap;
-  }
-
-  public String tagsToPath(Map tags) {
-    StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath);
-    for (String tagKey : tags.keySet()) {
-      stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey));
-    }
-    return stringBuilder.toString();
-  }
-
-  @Override
-  public void init() throws MetadataException {
-    if (!config.isEnableIDTableLogFile()
-        && config.getDeviceIDTransformationMethod().equals("SHA256")) {
-      throw new MetadataException(
-          "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\"");
-    }
-  }
-
-  @Override
-  public void clear() {
-    return;
-  }
-
-  @Override
-  public void forceMlog() {
-    return;
-  }
-
-  @Override
-  public SchemaRegionId getSchemaRegionId() {
-    return schemaRegionId;
-  }
-
-  @Override
-  public String getStorageGroupFullPath() {
-    return storageGroupFullPath;
-  }
-
-  @Override
-  public void deleteSchemaRegion() throws MetadataException {
-    return;
-  }
-
-  @Override
-  public boolean createSnapshot(File snapshotDir) {
-    return false;
-  }
-
-  @Override
-  public void loadSnapshot(File latestSnapshotRootDir) {
-    return;
-  }
-
-  private void createTagInvertedIndex(PartialPath devicePath) {
-    IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
-    Map tagsMap = pathToTags(devicePath.getFullPath());
-
-    deviceIDS.add(deviceID);
-
-    for (String tagkey : tagsMap.keySet()) {
-      String tagValue = tagsMap.get(tagkey);
-      Map> tagkeyMap =
-          tagInvertedIndex.computeIfAbsent(tagkey, key -> new HashMap<>());
-      List ids = tagkeyMap.computeIfAbsent(tagValue, key -> new ArrayList<>());
-      ids.add(deviceIDS.size() - 1);
-    }
-  }
-
-  private void createTimeseries(
-      PartialPath path,
-      TSDataType dataType,
-      TSEncoding encoding,
-      CompressionType compressor,
-      Map props)
-      throws MetadataException {
-    createTimeseries(
-        new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0);
-  }
-
-  private void createAlignedTimeSeries(
-      PartialPath prefixPath,
-      List measurements,
-      List dataTypes,
-      List encodings,
-      List compressors)
-      throws MetadataException {
-    createAlignedTimeSeries(
-        new CreateAlignedTimeSeriesPlan(
-            prefixPath, measurements, dataTypes, encodings, compressors, null, null, null));
-  }
-
-  @Override
-  public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {
-    PartialPath devicePath = plan.getPath().getDevicePath();
-    Map tags = pathToTags(devicePath.getFullPath());
-    PartialPath path = new PartialPath(tagsToPath(tags) + "." + plan.getPath().getMeasurement());
-    plan.setPath(path);
-    devicePath = plan.getPath().getDevicePath();
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath());
-    if (deviceEntry != null) {
-      if (deviceEntry.isAligned()) {
-        throw new AlignedTimeseriesException(
-            "Timeseries under this entity is not aligned, please use createTimeseries or change entity.",
-            devicePath.getFullPath() + "." + plan.getPath().getMeasurement());
-      } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) {
-        throw new PathAlreadyExistException(
-            devicePath.getFullPath() + "." + plan.getPath().getMeasurement());
-      }
-    }
-    idTable.createTimeseries(plan);
-    if (deviceEntry == null) {
-      createTagInvertedIndex(devicePath);
-    }
-  }
-
-  @Override
-  public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {
-    PartialPath devicePath = plan.getPrefixPath();
-    Map tags = pathToTags(devicePath.getFullPath());
-    PartialPath path = new PartialPath(tagsToPath(tags));
-    plan.setPrefixPath(path);
-    devicePath = plan.getPrefixPath();
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath());
-    if (deviceEntry != null) {
-      if (!deviceEntry.isAligned()) {
-        throw new AlignedTimeseriesException(
-            "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.",
-            devicePath.getFullPath());
-      } else {
-        List measurements = plan.getMeasurements();
-        List dataTypes = plan.getDataTypes();
-        List encodings = plan.getEncodings();
-        List compressors = plan.getCompressors();
-
-        List tmpMeasurements = new LinkedList<>();
-        List tmpDataTypes = new LinkedList<>();
-        List tmpEncodings = new LinkedList<>();
-        List tmpCompressors = new LinkedList<>();
-        for (int i = 0; i < measurements.size(); i++) {
-          String measurement = measurements.get(i);
-          if (!deviceEntry.getMeasurementMap().containsKey(measurement)) {
-            tmpMeasurements.add(measurements.get(i));
-            tmpDataTypes.add(dataTypes.get(i));
-            tmpEncodings.add(encodings.get(i));
-            tmpCompressors.add(compressors.get(i));
-          }
-        }
-        if (tmpMeasurements.size() == 0)
-          throw new PathAlreadyExistException(devicePath.getFullPath());
-        plan.setMeasurements(tmpMeasurements);
-        plan.setDataTypes(tmpDataTypes);
-        plan.setEncodings(tmpEncodings);
-        plan.setCompressors(tmpCompressors);
-      }
-    }
-    idTable.createAlignedTimeseries(plan);
-    if (deviceEntry == null) {
-      createTagInvertedIndex(devicePath);
-    }
-  }
-
-  @Override
-  public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch)
-      throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException {
-    return 0;
-  }
-
-  @Override
-  public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {}
-
-  @Override
-  public List fetchSchemaBlackList(PathPatternTree patternTree)
-      throws MetadataException {
-    return null;
-  }
-
-  @Override
-  public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {}
-
-  @Override
-  public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public boolean isPathExist(PartialPath path) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch)
-      throws MetadataException {
-    int res = 0;
-    List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern);
-    for (IDeviceID deviceID : deviceIDs) {
-      res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size();
-    }
-    return res;
-  }
-
-  @Override
-  public int getAllTimeseriesCount(
-      PartialPath pathPattern, Map templateMap, boolean isPrefixMatch)
-      throws MetadataException {
-    return 0;
-  }
-
-  @Override
-  public int getAllTimeseriesCount(
-      PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains)
-      throws MetadataException {
-    return 0;
-  }
-
-  @Override
-  public Map getMeasurementCountGroupByLevel(
-      PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public Map getMeasurementCountGroupByLevel(
-      PartialPath pathPattern,
-      int level,
-      boolean isPrefixMatch,
-      String key,
-      String value,
-      boolean isContains)
-      throws MetadataException {
-    return null;
-  }
-
-  @Override
-  public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch)
-      throws MetadataException {
-    if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) {
-      return deviceIDS.size();
-    } else {
-      return getDeviceIDsByInvertedIndex(pathPattern).size();
-    }
-  }
-
-  @Override
-  public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch)
-      throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public List getNodesListInGivenLevel(
-      PartialPath pathPattern,
-      int nodeLevel,
-      boolean isPrefixMatch,
-      LocalSchemaProcessor.StorageGroupFilter filter)
-      throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public Set getChildNodePathInNextLevel(PartialPath pathPattern)
-      throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public Set getBelongedDevices(PartialPath timeseries) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch)
-      throws MetadataException {
-    List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern);
-    Set res = new HashSet<>();
-    String devicePath = pathPattern.getFullPath();
-    if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) {
-      DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath);
-      if (deviceEntry != null) {
-        res.add(pathPattern);
-      }
-      return res;
-    }
-    for (IDeviceID deviceID : deviceIDs) {
-      if (deviceID instanceof SHA256DeviceID) {
-        DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID());
-        Map map = deviceEntry.getMeasurementMap();
-        for (String m : map.keySet()) {
-          SchemaEntry schemaEntry = map.get(m);
-          List schemaEntries = new ArrayList<>();
-          schemaEntries.add(schemaEntry);
-          List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries);
-          DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0);
-          res.add(
-              new PartialPath(
-                  diskSchemaEntry.seriesKey.substring(
-                      0,
-                      diskSchemaEntry.seriesKey.length()
-                          - diskSchemaEntry.measurementName.length()
-                          - 1)));
-          break;
-        }
-      } else {
-        res.add(new PartialPath(deviceID.toStringID()));
-      }
-    }
-    return res;
-  }
-
-  @Override
-  public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan)
-      throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch)
-      throws MetadataException {
-    PartialPath devicePath = pathPattern.getDevicePath();
-    if (devicePath.getFullPath().endsWith(TAIL)) {
-      return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch);
-    } else {
-      return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch);
-    }
-  }
-
-  private List getMeasurementPathsWithPointQuery(
-      PartialPath devicePath, boolean isPrefixMatch) throws MetadataException {
-    List res = new LinkedList<>();
-    String path = devicePath.getFullPath();
-    Map tags = pathToTags(path);
-    path = tagsToPath(tags);
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(path);
-    if (deviceEntry == null) return res;
-    Map schemaMap = deviceEntry.getMeasurementMap();
-    for (String measurement : schemaMap.keySet()) {
-      SchemaEntry schemaEntry = schemaMap.get(measurement);
-      MeasurementPath measurementPath =
-          new MeasurementPath(
-              path,
-              measurement,
-              new MeasurementSchema(
-                  measurement,
-                  schemaEntry.getTSDataType(),
-                  schemaEntry.getTSEncoding(),
-                  schemaEntry.getCompressionType()));
-      measurementPath.setUnderAlignedEntity(deviceEntry.isAligned());
-      res.add(measurementPath);
-    }
-
-    return res;
-  }
-
-  private List getMeasurementPathsWithBatchQuery(
-      PartialPath devicePath, boolean isPrefixMatch) throws MetadataException {
-    List res = new LinkedList<>();
-    List deviceIDs = getDeviceIdFromInvertedIndex(devicePath);
-    for (IDeviceID deviceID : deviceIDs) {
-      DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID());
-      Map schemaMap = deviceEntry.getMeasurementMap();
-      if (deviceID instanceof SHA256DeviceID) {
-        for (String measurement : schemaMap.keySet()) {
-          SchemaEntry schemaEntry = schemaMap.get(measurement);
-          List schemaEntries = new ArrayList<>();
-          schemaEntries.add(schemaEntry);
-          List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries);
-          DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0);
-          MeasurementPath measurementPath =
-              new MeasurementPath(
-                  new PartialPath(diskSchemaEntry.seriesKey),
-                  new MeasurementSchema(
-                      measurement,
-                      schemaEntry.getTSDataType(),
-                      schemaEntry.getTSEncoding(),
-                      schemaEntry.getCompressionType()));
-          measurementPath.setUnderAlignedEntity(deviceEntry.isAligned());
-          res.add(measurementPath);
-        }
-      } else {
-        for (String measurement : schemaMap.keySet()) {
-          SchemaEntry schemaEntry = schemaMap.get(measurement);
-          MeasurementPath measurementPath =
-              new MeasurementPath(
-                  deviceID.toStringID(),
-                  measurement,
-                  new MeasurementSchema(
-                      measurement,
-                      schemaEntry.getTSDataType(),
-                      schemaEntry.getTSEncoding(),
-                      schemaEntry.getCompressionType()));
-          measurementPath.setUnderAlignedEntity(deviceEntry.isAligned());
-          res.add(measurementPath);
-        }
-      }
-    }
-    return res;
-  }
-
-  @Override
-  public Pair, Integer> getMeasurementPathsWithAlias(
-      PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch)
-      throws MetadataException {
-    List res = getMeasurementPaths(pathPattern, isPrefixMatch);
-    Pair, Integer> result = new Pair<>(res, 0);
-    return result;
-  }
-
-  @Override
-  public List fetchSchema(
-      PartialPath pathPattern, Map templateMap) throws MetadataException {
-    return null;
-  }
-
-  @Override
-  public Pair, Integer> showTimeseries(
-      ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException {
-    List res = new ArrayList<>();
-    Pair, Integer> result = new Pair<>(res, 0);
-    String path = plan.getPath().getFullPath();
-    if (!path.endsWith(TAIL)) {
-      Map tags = pathToTags(path);
-      path = tagsToPath(tags);
-      DeviceEntry deviceEntry = idTable.getDeviceEntry(path);
-      if (deviceEntry != null) {
-        Map measurementMap = deviceEntry.getMeasurementMap();
-        for (String m : measurementMap.keySet()) {
-          SchemaEntry schemaEntry = measurementMap.get(m);
-          res.add(
-              new ShowTimeSeriesResult(
-                  path + "." + m,
-                  "null",
-                  storageGroupFullPath,
-                  schemaEntry.getTSDataType(),
-                  schemaEntry.getTSEncoding(),
-                  schemaEntry.getCompressionType(),
-                  schemaEntry.getLastTime(),
-                  new HashMap<>(),
-                  new HashMap<>()));
-        }
-      }
-      return result;
-    }
-    List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath());
-    for (IDeviceID deviceID : deviceIDs) {
-      getTimeSeriesResultOfDeviceFromIDTable(res, deviceID);
-    }
-    return result;
-  }
-
-  private List getDeviceIdFromInvertedIndex(PartialPath devicePath)
-      throws MetadataException {
-    String path = devicePath.getFullPath();
-    if (path.endsWith(TAIL)) {
-      path = path.substring(0, path.length() - TAIL.length());
-      devicePath = new PartialPath(path);
-    }
-    if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) {
-      return deviceIDS;
-    } else {
-      List res = new LinkedList<>();
-      List ids = getDeviceIDsByInvertedIndex(devicePath);
-      if (ids.size() > 0) {
-        for (int id : ids) {
-          res.add(deviceIDS.get(id));
-        }
-      }
-      return res;
-    }
-  }
-
-  private void getTimeSeriesResultOfDeviceFromIDTable(
-      List res, IDeviceID deviceID) {
-    Map measurementMap =
-        idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap();
-    if (deviceID instanceof SHA256DeviceID) {
-      for (String m : measurementMap.keySet()) {
-        SchemaEntry schemaEntry = measurementMap.get(m);
-        List schemaEntries = new ArrayList<>();
-        schemaEntries.add(schemaEntry);
-        List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries);
-        DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0);
-        res.add(
-            new ShowTimeSeriesResult(
-                diskSchemaEntry.seriesKey,
-                "null",
-                storageGroupFullPath,
-                schemaEntry.getTSDataType(),
-                schemaEntry.getTSEncoding(),
-                schemaEntry.getCompressionType(),
-                schemaEntry.getLastTime(),
-                new HashMap<>(),
-                new HashMap<>()));
-      }
-    } else {
-      for (String m : measurementMap.keySet()) {
-        SchemaEntry schemaEntry = measurementMap.get(m);
-        res.add(
-            new ShowTimeSeriesResult(
-                deviceID.toStringID() + "." + m,
-                "null",
-                storageGroupFullPath,
-                schemaEntry.getTSDataType(),
-                schemaEntry.getTSEncoding(),
-                schemaEntry.getCompressionType(),
-                schemaEntry.getLastTime(),
-                new HashMap<>(),
-                new HashMap<>()));
-      }
-    }
-  }
-
-  private List getDeviceIDsByInvertedIndex(PartialPath path) {
-    Map tags = pathToTags(path.getFullPath());
-    List idsCollection = new ArrayList<>(tags.keySet().size());
-    for (String tagKey : tags.keySet()) {
-      if (!tagInvertedIndex.containsKey(tagKey)
-          || !tagInvertedIndex.get(tagKey).containsKey(tags.get(tagKey))) {
-        return new ArrayList<>();
-      }
-      List ids = tagInvertedIndex.get(tagKey).get(tags.get(tagKey));
-      idsCollection.add(new ArrayList(ids));
-    }
-    if (idsCollection.size() == 0) return new ArrayList<>();
-    List ids = idsCollection.get(0);
-    for (int i = 1; i < idsCollection.size(); i++) {
-      List list = idsCollection.get(i);
-      ids.retainAll(list);
-    }
-    return ids;
-  }
-
-  @Override
-  public List getAllMeasurementByDevicePath(PartialPath devicePath)
-      throws PathNotExistException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public IMNode getDeviceNode(PartialPath path) throws MetadataException {
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath());
-    if (deviceEntry == null) throw new PathNotExistException(path.getFullPath());
-    return new EntityMNode(storageGroupMNode, path.getFullPath());
-  }
-
-  @Override
-  public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void upsertTagsAndAttributes(
-      String alias,
-      Map tagsMap,
-      Map attributesMap,
-      PartialPath fullPath)
-      throws MetadataException, IOException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void addAttributes(Map attributesMap, PartialPath fullPath)
-      throws MetadataException, IOException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void addTags(Map tagsMap, PartialPath fullPath)
-      throws MetadataException, IOException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void dropTagsOrAttributes(Set keySet, PartialPath fullPath)
-      throws MetadataException, IOException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath)
-      throws MetadataException, IOException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath)
-      throws MetadataException, IOException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan)
-      throws MetadataException, IOException {
-    PartialPath devicePath = plan.getDevicePath();
-    Map tags = pathToTags(devicePath.getFullPath());
-    devicePath = new PartialPath(tagsToPath(tags));
-    plan.setDevicePath(devicePath);
-    String[] measurementList = plan.getMeasurements();
-    IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes();
-    checkAlignedAndAutoCreateSeries(plan);
-    IMNode deviceMNode = getDeviceNode(devicePath);
-    IMeasurementMNode measurementMNode;
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath());
-    Map schemaMap = deviceEntry.getMeasurementMap();
-    for (int i = 0; i < measurementList.length; i++) {
-      SchemaEntry schemaEntry = schemaMap.get(measurementList[i]);
-      measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null);
-      // check type is match
-      try {
-        SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType());
-      } catch (DataTypeMismatchException mismatchException) {
-        if (!config.isEnablePartialInsert()) {
-          throw mismatchException;
-        } else {
-          // mark failed measurement
-          plan.markFailedMeasurementInsertion(i, mismatchException);
-          continue;
-        }
-      }
-      measurementMNodes[i] = measurementMNode;
-    }
-    plan.setDeviceID(deviceEntry.getDeviceID());
-    plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false));
-    return deviceMNode;
-  }
-
-  @Override
-  public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate(
-      PartialPath devicePath,
-      String[] measurements,
-      Function getDataType,
-      TSEncoding[] encodings,
-      CompressionType[] compressionTypes,
-      boolean aligned)
-      throws MetadataException {
-    List measurementSchemaInfoList = new ArrayList<>(measurements.length);
-    for (int i = 0; i < measurements.length; i++) {
-      SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]);
-      if (schemaEntry == null) {
-        if (config.isAutoCreateSchemaEnabled()) {
-          if (aligned) {
-            TSDataType dataType = getDataType.apply(i);
-            internalAlignedCreateTimeseries(
-                devicePath,
-                Collections.singletonList(measurements[i]),
-                Collections.singletonList(dataType),
-                Collections.singletonList(
-                    encodings[i] == null ? getDefaultEncoding(dataType) : encodings[i]),
-                Collections.singletonList(
-                    compressionTypes[i] == null
-                        ? TSFileDescriptor.getInstance().getConfig().getCompressor()
-                        : compressionTypes[i]));
-
-          } else {
-            internalCreateTimeseries(
-                devicePath.concatNode(measurements[i]),
-                getDataType.apply(i),
-                encodings[i],
-                compressionTypes[i]);
-          }
-        }
-        schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]);
-      }
-      measurementSchemaInfoList.add(
-          new MeasurementSchemaInfo(
-              measurements[i],
-              new MeasurementSchema(
-                  measurements[i],
-                  schemaEntry.getTSDataType(),
-                  schemaEntry.getTSEncoding(),
-                  schemaEntry.getCompressionType()),
-              null));
-    }
-    return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList);
-  }
-
-  private SchemaEntry getSchemaEntry(String devicePath, String measurementName) {
-    DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath);
-    if (deviceEntry == null) return null;
-    return deviceEntry.getSchemaEntry(measurementName);
-  }
-
-  private DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate(
-      PartialPath devicePath,
-      String[] measurements,
-      Function getDataType,
-      boolean aligned)
-      throws MetadataException {
-    List measurementSchemaInfoList = new ArrayList<>(measurements.length);
-    for (int i = 0; i < measurements.length; i++) {
-      SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]);
-      if (schemaEntry == null) {
-        if (config.isAutoCreateSchemaEnabled()) {
-          if (aligned) {
-            internalAlignedCreateTimeseries(
-                devicePath,
-                Collections.singletonList(measurements[i]),
-                Collections.singletonList(getDataType.apply(i)));
-
-          } else {
-            internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i));
-          }
-        }
-        schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]);
-      }
-      measurementSchemaInfoList.add(
-          new MeasurementSchemaInfo(
-              measurements[i],
-              new MeasurementSchema(
-                  measurements[i],
-                  schemaEntry.getTSDataType(),
-                  schemaEntry.getTSEncoding(),
-                  schemaEntry.getCompressionType()),
-              null));
-    }
-    return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList);
-  }
-
-  private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException {
-    String[] measurementList = plan.getMeasurements();
-    try {
-      if (plan.isAligned()) {
-        internalAlignedCreateTimeseries(
-            plan.getDevicePath(),
-            Arrays.asList(measurementList),
-            Arrays.asList(plan.getDataTypes()));
-      } else {
-        internalCreateTimeseries(
-            plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]);
-      }
-    } catch (MetadataException e) {
-      if (!(e instanceof PathAlreadyExistException)) {
-        throw e;
-      }
-    }
-  }
-
-  private void internalCreateTimeseries(PartialPath path, TSDataType dataType)
-      throws MetadataException {
-    createTimeseries(
-        path,
-        dataType,
-        getDefaultEncoding(dataType),
-        TSFileDescriptor.getInstance().getConfig().getCompressor(),
-        Collections.emptyMap());
-  }
-
-  /** create timeseries ignoring PathAlreadyExistException */
-  private void internalCreateTimeseries(
-      PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor)
-      throws MetadataException {
-    if (encoding == null) {
-      encoding = getDefaultEncoding(dataType);
-    }
-    if (compressor == null) {
-      compressor = TSFileDescriptor.getInstance().getConfig().getCompressor();
-    }
-    createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap());
-  }
-
-  private void internalAlignedCreateTimeseries(
-      PartialPath prefixPath, List measurements, List dataTypes)
-      throws MetadataException {
-    List encodings = new ArrayList<>();
-    List compressors = new ArrayList<>();
-    for (TSDataType dataType : dataTypes) {
-      encodings.add(getDefaultEncoding(dataType));
-      compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor());
-    }
-    createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors);
-  }
-
-  /** create aligned timeseries ignoring PathAlreadyExistException */
-  private void internalAlignedCreateTimeseries(
-      PartialPath prefixPath,
-      List measurements,
-      List dataTypes,
-      List encodings,
-      List compressors)
-      throws MetadataException {
-    createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors);
-  }
-
-  @Override
-  public Set getPathsSetTemplate(String templateName) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public Set getPathsUsingTemplate(String templateName) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public boolean isTemplateAppendable(Template template, List measurements)
-      throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template)
-      throws MetadataException {}
-
-  @Override
-  public List getPathsUsingTemplate(int templateId) throws MetadataException {
-    return null;
-  }
-
-  @Override
-  public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-
-  @Override
-  public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException {
-    throw new UnsupportedOperationException("");
-  }
-}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
index 8a01a75fe2a68..ac0611fbca2cc 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
@@ -23,6 +23,7 @@
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.exception.MetadataException;
 import org.apache.iotdb.commons.file.SystemFileFactory;
+import org.apache.iotdb.commons.path.MeasurementPath;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.path.PathPatternTree;
 import org.apache.iotdb.commons.utils.TestOnly;
@@ -44,7 +45,13 @@
 import org.apache.iotdb.db.metadata.mnode.IMNode;
 import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
 import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode;
-import org.apache.iotdb.db.metadata.path.MeasurementPath;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.IActivateTemplateInClusterPlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.IActivateTemplatePlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.IAutoCreateDeviceMNodePlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateAlignedTimeSeriesPlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateTimeSeriesPlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.ISetTemplatePlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.IUnsetTemplatePlan;
 import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion;
 import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.idtable.IDTableWithDeviceIDListImpl;
@@ -56,15 +63,10 @@
 import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo;
 import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo;
 import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
-import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan;
-import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan;
-import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan;
 import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan;
 import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan;
 import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan;
 import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan;
 import org.apache.iotdb.db.query.context.QueryContext;
 import org.apache.iotdb.db.query.dataset.ShowDevicesResult;
 import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult;
@@ -237,7 +239,7 @@ private void createAlignedTimeSeries(
   }
 
   @Override
-  public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException {
+  public void createTimeseries(ICreateTimeSeriesPlan plan, long offset) throws MetadataException {
     PartialPath devicePath = plan.getPath().getDevicePath();
     PartialPath path =
         new PartialPath(
@@ -265,14 +267,14 @@ public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws Meta
   }
 
   @Override
-  public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException {
-    PartialPath devicePath = plan.getPrefixPath();
+  public void createAlignedTimeSeries(ICreateAlignedTimeSeriesPlan plan) throws MetadataException {
+    PartialPath devicePath = plan.getDevicePath();
     PartialPath path =
         new PartialPath(
             PathTagConverterUtils.pathToTagsSortPath(
                 storageGroupFullPath, devicePath.getFullPath()));
-    plan.setPrefixPath(path);
-    devicePath = plan.getPrefixPath();
+    plan.setDevicePath(path);
+    devicePath = plan.getDevicePath();
     DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath.getFullPath());
     if (deviceEntry != null) {
       if (!deviceEntry.isAligned()) {
@@ -292,8 +294,14 @@ public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws Met
     }
   }
 
+  @Override
+  public Map checkMeasurementExistence(
+      PartialPath devicePath, List measurementList, List aliasList) {
+    throw new UnsupportedOperationException("checkMeasurementExistence");
+  }
+
   private void filterExistingMeasurements(
-      CreateAlignedTimeSeriesPlan plan, Set measurementSet) {
+      ICreateAlignedTimeSeriesPlan plan, Set measurementSet) {
     List measurements = plan.getMeasurements();
     List dataTypes = plan.getDataTypes();
     List encodings = plan.getEncodings();
@@ -335,7 +343,7 @@ public void rollbackSchemaBlackList(PathPatternTree patternTree) throws Metadata
   }
 
   @Override
-  public List fetchSchemaBlackList(PathPatternTree patternTree)
+  public Set fetchSchemaBlackList(PathPatternTree patternTree)
       throws MetadataException {
     throw new UnsupportedOperationException("fetchSchemaBlackList");
   }
@@ -346,9 +354,7 @@ public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws Meta
   }
 
   @Override
-  public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException {
-    throw new UnsupportedOperationException("autoCreateDeviceMNode");
-  }
+  public void autoCreateDeviceMNode(IAutoCreateDeviceMNodePlan plan) throws MetadataException {}
 
   @Override
   public boolean isPathExist(PartialPath path) throws MetadataException {
@@ -540,8 +546,8 @@ public Pair, Integer> getMatchedDevices(ShowDevicesPlan
   }
 
   @Override
-  public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch)
-      throws MetadataException {
+  public List getMeasurementPaths(
+      PartialPath pathPattern, boolean isPrefixMatch, boolean withTags) throws MetadataException {
     PartialPath devicePath = pathPattern.getDevicePath();
     if (devicePath.getFullPath().endsWith(TAIL)) {
       return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch);
@@ -550,6 +556,22 @@ public List getMeasurementPaths(PartialPath pathPattern, boolea
     }
   }
 
+  @Override
+  public Pair, Integer> getMeasurementPathsWithAlias(
+      PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch, boolean withTags)
+      throws MetadataException {
+    List res = getMeasurementPaths(pathPattern, isPrefixMatch, false);
+    Pair, Integer> result = new Pair<>(res, 0);
+    return result;
+  }
+
+  @Override
+  public List fetchSchema(
+      PartialPath pathPattern, Map templateMap, boolean withTags)
+      throws MetadataException {
+    return null;
+  }
+
   private List getMeasurementPathsWithPointQuery(
       PartialPath devicePath, boolean isPrefixMatch) throws MetadataException {
     List measurementPaths = new LinkedList<>();
@@ -573,21 +595,6 @@ private List getMeasurementPathsWithBatchQuery(
     return getMeasurementPaths(deviceIDs);
   }
 
-  @Override
-  public Pair, Integer> getMeasurementPathsWithAlias(
-      PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch)
-      throws MetadataException {
-    List res = getMeasurementPaths(pathPattern, isPrefixMatch);
-    Pair, Integer> result = new Pair<>(res, 0);
-    return result;
-  }
-
-  @Override
-  public List fetchSchema(
-      PartialPath pathPattern, Map templateMap) throws MetadataException {
-    throw new UnsupportedOperationException("fetchSchema");
-  }
-
   @Override
   public Pair, Integer> showTimeseries(
       ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException {
@@ -907,22 +914,22 @@ public boolean isTemplateAppendable(Template template, List measurements
   }
 
   @Override
-  public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException {
+  public void setSchemaTemplate(ISetTemplatePlan plan) throws MetadataException {
     throw new UnsupportedOperationException("setSchemaTemplate");
   }
 
   @Override
-  public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException {
+  public void unsetSchemaTemplate(IUnsetTemplatePlan plan) throws MetadataException {
     throw new UnsupportedOperationException("unsetSchemaTemplate");
   }
 
   @Override
-  public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException {
+  public void setUsingSchemaTemplate(IActivateTemplatePlan plan) throws MetadataException {
     throw new UnsupportedOperationException("setUsingSchemaTemplate");
   }
 
   @Override
-  public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template)
+  public void activateSchemaTemplate(IActivateTemplateInClusterPlan plan, Template template)
       throws MetadataException {
     throw new UnsupportedOperationException("activateSchemaTemplate");
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java
index 4f506d5461226..31b0df4888f3e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java
@@ -25,8 +25,8 @@
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
 import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
 import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry;
-import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan;
-import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateAlignedTimeSeriesPlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateTimeSeriesPlan;
 
 import java.io.File;
 import java.io.IOException;
@@ -66,9 +66,9 @@ private boolean deviceEntryNotExist(String devicePath) {
    * @throws MetadataException
    */
   @Override
-  public synchronized void createAlignedTimeseries(CreateAlignedTimeSeriesPlan plan)
+  public synchronized void createAlignedTimeseries(ICreateAlignedTimeSeriesPlan plan)
       throws MetadataException {
-    String devicePath = plan.getPrefixPath().getFullPath();
+    String devicePath = plan.getDevicePath().getFullPath();
     // This device path is created for the first time, append it to deviceIdList
     if (deviceEntryNotExist(devicePath)) {
       IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath);
@@ -84,7 +84,7 @@ public synchronized void createAlignedTimeseries(CreateAlignedTimeSeriesPlan pla
    * @throws MetadataException
    */
   @Override
-  public synchronized void createTimeseries(CreateTimeSeriesPlan plan) throws MetadataException {
+  public synchronized void createTimeseries(ICreateTimeSeriesPlan plan) throws MetadataException {
     String devicePath = plan.getPath().getDevicePath().getFullPath();
     // This device path is created for the first time, append it to deviceIdList
     if (deviceEntryNotExist(devicePath)) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java
index b6f5402a7169f..d83c016ee0c62 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java
@@ -19,10 +19,10 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.utils;
 
 import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.MeasurementPath;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry;
 import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry;
-import org.apache.iotdb.db.metadata.path.MeasurementPath;
 import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
 import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
 import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java
index 513d04783b1ff..5f4ddb78f34dc 100644
--- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java
+++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java
@@ -19,10 +19,10 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion;
 
 import org.apache.iotdb.commons.consensus.SchemaRegionId;
+import org.apache.iotdb.commons.path.MeasurementPath;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.utils.FileUtils;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.metadata.path.MeasurementPath;
 import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan;
 import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
 import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
@@ -150,24 +150,24 @@ public void testGetMeasurementPaths() {
   private void getMeasurementPathsTest() throws Exception {
     PartialPath pathPattern = new PartialPath(storageGroup + ".tag1.a.tag2.b.s0");
     List measurementPaths =
-        tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+        tagSchemaRegion.getMeasurementPaths(pathPattern, false, false);
     assertEquals(measurementPaths.size(), 1);
     assertEquals(measurementPaths.get(0).getFullPath(), storageGroup + ".tag1.a.tag2.b.s0");
 
     pathPattern = new PartialPath(storageGroup + ".tag1.a.**.s");
-    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false, false);
     assertEquals(measurementPaths.size(), 5);
 
     pathPattern = new PartialPath(storageGroup + ".tag2.b.**.s");
-    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false, false);
     assertEquals(measurementPaths.size(), 1);
 
     pathPattern = new PartialPath(storageGroup + ".tag3.b.**.s");
-    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false, false);
     assertEquals(measurementPaths.size(), 3);
 
     pathPattern = new PartialPath(storageGroup + ".tag2.y.tag1.x.**.s");
-    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false);
+    measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false, false);
     assertEquals(measurementPaths.size(), 3);
   }
 
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
index 66d7781fed87f..39808bbf950f1 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
@@ -23,7 +23,6 @@
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.utils.PathUtils;
 import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.metadata.path.MeasurementPath;
 import org.apache.iotdb.db.mpp.common.schematree.node.SchemaEntityNode;
 import org.apache.iotdb.db.mpp.common.schematree.node.SchemaInternalNode;
 import org.apache.iotdb.db.mpp.common.schematree.node.SchemaMeasurementNode;

From afe53ec2cb792af10bf05c2408ea3fd7f2bd4997 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sat, 22 Oct 2022 14:58:44 +0800
Subject: [PATCH 36/53] Separate request and context

---
 .../tagIndex/ITagInvertedIndex.java           | 12 +--
 .../tagIndex/Request/DeletionRequest.java     | 65 +++++++++++++++
 .../tagIndex/Request/InsertionRequest.java    | 69 ++++++++++++++++
 .../tagIndex/Request/QueryRequest.java        | 59 ++++++++++++++
 .../tagIndex/Request/RequestStatus.java       | 40 ++++++++++
 .../tagIndex/TagInvertedIndex.java            | 40 ++++++----
 .../tagIndex/deletion/DeletionManager.java    | 15 ++--
 .../tagIndex/deletion/MemChunkDeletion.java   | 10 ++-
 .../deletion/MemChunkGroupDeletion.java       | 14 ++--
 .../tagIndex/deletion/MemTableDeletion.java   | 17 ++--
 .../tagIndex/insertion/InsertionManager.java  | 15 ++--
 .../insertion/MemChunkGroupInsertion.java     | 14 ++--
 .../tagIndex/insertion/MemChunkInsertion.java | 11 ++-
 .../tagIndex/insertion/MemTableInsertion.java | 14 ++--
 .../tagIndex/query/MemChunkGroupQuery.java    | 11 ++-
 .../tagIndex/query/MemChunkQuery.java         | 12 +--
 .../tagIndex/query/MemTableQuery.java         | 12 +--
 .../tagIndex/query/QueryManager.java          |  6 +-
 .../tagIndex/recover/RecoverManager.java      | 20 +++--
 .../tagIndex/wal/WALManager.java              | 79 ++++++++-----------
 .../lsm/context/DeleteRequestContext.java     | 45 -----------
 .../lsm/context/FlushRequestContext.java      |  1 -
 .../lsm/context/InsertRequestContext.java     | 41 ----------
 .../lsm/context/QueryRequestContext.java      | 20 +----
 .../iotdb/lsm/context/RequestContext.java     | 35 --------
 .../lsm/levelProcess/BasicLevelProcess.java   | 18 ++---
 .../lsm/levelProcess/DeleteLevelProcess.java  | 10 +--
 .../lsm/levelProcess/FlushLevelProcess.java   |  5 +-
 .../lsm/levelProcess/InsertLevelProcess.java  | 10 +--
 .../iotdb/lsm/levelProcess/LevelProcess.java  |  6 +-
 .../lsm/levelProcess/LevelProcessChain.java   | 10 +--
 .../lsm/levelProcess/QueryLevelProcess.java   |  9 ++-
 .../iotdb/lsm/manager/BasicLSMManager.java    | 20 ++---
 .../apache/iotdb/lsm/manager/LSMManager.java  |  8 +-
 .../iotdb/lsm/request/DeletionRequest.java    | 28 +++++++
 .../iotdb/lsm/request/InsertionRequest.java   | 28 +++++++
 .../iotdb/lsm/request/QueryRequest.java       | 26 ++++++
 .../org/apache/iotdb/lsm/request/Request.java | 48 +++++++++++
 .../lsm/{context => request}/RequestType.java |  2 +-
 .../iotdb/lsm/strategy/AccessStrategy.java    |  4 +-
 .../iotdb/lsm/strategy/BFSAccessStrategy.java | 14 ++--
 .../lsm/strategy/PostOrderAccessStrategy.java | 10 +--
 .../lsm/strategy/PreOrderAccessStrategy.java  | 10 +--
 .../lsm/strategy/RBFSAccessStrategy.java      | 16 ++--
 .../tagSchemaRegion/TagSchemaRegionTest.java  |  2 +-
 45 files changed, 612 insertions(+), 349 deletions(-)
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/{context => request}/RequestType.java (95%)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java
index 18c90ada1550e..2b52195e2cef0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java
@@ -18,8 +18,8 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
 
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 
 import java.util.List;
 import java.util.Map;
@@ -30,9 +30,9 @@ public interface ITagInvertedIndex {
   /**
    * insert tags and id using insert request context
    *
-   * @param context insert request context
+   * @param insertionRequest insert request context
    */
-  void addTags(InsertRequestContext context);
+  void addTags(InsertionRequest insertionRequest);
 
   /**
    * insert tags and device id
@@ -45,9 +45,9 @@ public interface ITagInvertedIndex {
   /**
    * delete tags and id using delete request context
    *
-   * @param context delete request context
+   * @param deletionRequest delete request context
    */
-  void removeTags(DeleteRequestContext context);
+  void removeTags(DeletionRequest deletionRequest);
 
   /**
    * delete tags and id using delete request context
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
new file mode 100644
index 0000000000000..d35738e6c9157
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
+
+import org.apache.iotdb.lsm.context.RequestContext;
+
+import java.util.List;
+
+public class DeletionRequest
+    extends org.apache.iotdb.lsm.request.DeletionRequest {
+
+  List keys;
+
+  int value;
+
+  RequestStatus requestStatus;
+
+  public DeletionRequest(List keys, int value) {
+    super();
+    this.keys = keys;
+    this.value = value;
+    requestStatus = new RequestStatus();
+  }
+
+  @Override
+  public String getKey(RequestContext context) {
+    return keys.get(context.getLevel());
+  }
+
+  @Override
+  public RequestStatus getResult() {
+    return requestStatus;
+  }
+
+  @Override
+  public void setResult(RequestStatus result) {
+    requestStatus = result;
+  }
+
+  @Override
+  public List getKeys() {
+    return keys;
+  }
+
+  @Override
+  public Integer getValue() {
+    return value;
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
new file mode 100644
index 0000000000000..52afb9b1a339f
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
+
+import org.apache.iotdb.lsm.context.RequestContext;
+
+import java.util.List;
+
+public class InsertionRequest
+    extends org.apache.iotdb.lsm.request.InsertionRequest {
+
+  List keys;
+
+  int value;
+
+  RequestStatus requestStatus;
+
+  public InsertionRequest() {
+    super();
+  }
+
+  public InsertionRequest(List keys, int value) {
+    super();
+    this.keys = keys;
+    this.value = value;
+    requestStatus = new RequestStatus();
+  }
+
+  @Override
+  public String getKey(RequestContext context) {
+    return keys.get(context.getLevel());
+  }
+
+  @Override
+  public RequestStatus getResult() {
+    return requestStatus;
+  }
+
+  @Override
+  public void setResult(RequestStatus result) {
+    requestStatus = result;
+  }
+
+  @Override
+  public List getKeys() {
+    return keys;
+  }
+
+  @Override
+  public Integer getValue() {
+    return value;
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
new file mode 100644
index 0000000000000..18f816d265bbf
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
+
+import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.request.Request;
+
+import org.roaringbitmap.RoaringBitmap;
+
+import java.util.List;
+
+public class QueryRequest extends Request {
+
+  List keys;
+
+  RoaringBitmap roaringBitmap;
+
+  public QueryRequest(List keys) {
+    super();
+    this.keys = keys;
+    roaringBitmap = new RoaringBitmap();
+  }
+
+  @Override
+  public String getKey(RequestContext context) {
+    return keys.get(context.getLevel());
+  }
+
+  @Override
+  public RoaringBitmap getResult() {
+    return roaringBitmap;
+  }
+
+  @Override
+  public void setResult(RoaringBitmap result) {
+    roaringBitmap = result;
+  }
+
+  @Override
+  public List getKeys() {
+    return keys;
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java
new file mode 100644
index 0000000000000..185ecd7bd2389
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
+
+public class RequestStatus {
+
+  Boolean success;
+
+  public RequestStatus() {
+    success = true;
+  }
+
+  public RequestStatus(Boolean success) {
+    this.success = success;
+  }
+
+  public Boolean getSuccess() {
+    return success;
+  }
+
+  public void setSuccess(Boolean success) {
+    this.success = success;
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index b4b89baf44160..939680e50fd2f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -21,6 +21,9 @@
 import org.apache.iotdb.commons.utils.TestOnly;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
@@ -96,11 +99,11 @@ public synchronized void recover() {
   /**
    * insert tags and id using insert request context
    *
-   * @param context insert request context
+   * @param insertionRequest insert request context
    */
   @Override
-  public synchronized void addTags(InsertRequestContext context) {
-    int id = (int) context.getValue();
+  public synchronized void addTags(InsertionRequest insertionRequest) {
+    int id = insertionRequest.getValue();
     // if the device id can not be saved to the current working MemTable
     if (!inWorkingMemTable(id)) {
       workingMemTable.setStatus(MemTable.IMMUTABLE);
@@ -110,7 +113,7 @@ public synchronized void addTags(InsertRequestContext context) {
     MemTable memTable = workingMemTable;
     maxDeviceID = id;
     try {
-      insertionManager.process(memTable, context);
+      insertionManager.process(memTable, insertionRequest, new InsertRequestContext());
     } catch (Exception e) {
       logger.error(e.getMessage());
     }
@@ -144,11 +147,11 @@ public synchronized void addTags(Map tags, int id) {
   /**
    * delete tags and id using delete request context
    *
-   * @param context delete request context
+   * @param deletionRequest delete request context
    */
   @Override
-  public void removeTags(DeleteRequestContext context) {
-    int id = (int) context.getValue();
+  public void removeTags(DeletionRequest deletionRequest) {
+    int id = deletionRequest.getValue();
     MemTable memTable = null;
     if (inWorkingMemTable(id)) {
       memTable = workingMemTable;
@@ -156,7 +159,7 @@ public void removeTags(DeleteRequestContext context) {
       memTable = immutableMemTables.get(id / numOfDeviceIdsInMemTable);
     }
     try {
-      deletionManager.process(memTable, context);
+      deletionManager.process(memTable, deletionRequest, new DeleteRequestContext());
     } catch (Exception e) {
       logger.error(e.getMessage());
     }
@@ -238,26 +241,33 @@ private boolean inWorkingMemTable(int id) {
     return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable;
   }
 
+  private List generateKeys(String tagKey, String tagValue) {
+    List keys = new ArrayList<>();
+    keys.add(tagKey);
+    keys.add(tagValue);
+    return keys;
+  }
+
   private void addTag(MemTable memTable, String tagKey, String tagValue, int id) throws Exception {
-    InsertRequestContext insertContext = new InsertRequestContext(id, tagKey, tagValue);
-    insertionManager.process(memTable, insertContext);
+    InsertionRequest insertionRequest = new InsertionRequest(generateKeys(tagKey, tagValue), id);
+    insertionManager.process(memTable, insertionRequest, new InsertRequestContext());
   }
 
   private void removeTag(List memTables, String tagKey, String tagValue, int id)
       throws Exception {
-    DeleteRequestContext deleteContext = new DeleteRequestContext(id, tagKey, tagValue);
+    DeletionRequest deletionRequest = new DeletionRequest(generateKeys(tagKey, tagValue), id);
     for (MemTable memTable : memTables) {
-      deletionManager.process(memTable, deleteContext);
+      deletionManager.process(memTable, deletionRequest, new DeleteRequestContext());
     }
   }
 
   private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue)
       throws Exception {
-    QueryRequestContext queryContext = new QueryRequestContext(tagKey, tagValue);
+    QueryRequest queryRequest = new QueryRequest(generateKeys(tagKey, tagValue));
     for (MemTable memTable : memTables) {
-      queryManager.process(memTable, queryContext);
+      queryManager.process(memTable, queryRequest, new QueryRequestContext());
     }
-    return (RoaringBitmap) queryContext.getResult();
+    return queryRequest.getResult();
   }
 
   @TestOnly
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
index ff2911ed9cfe2..7297f46c2a5a1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
@@ -25,7 +26,8 @@
 import org.apache.iotdb.lsm.manager.BasicLSMManager;
 
 /** manage deletion to MemTable */
-public class DeletionManager extends BasicLSMManager {
+public class DeletionManager
+    extends BasicLSMManager {
 
   // use wal manager object to write wal file on deletion
   private WALManager walManager;
@@ -43,15 +45,16 @@ public DeletionManager(WALManager walManager) {
    * @throws Exception
    */
   @Override
-  public void preProcess(MemTable root, DeleteRequestContext context) throws Exception {
-    if (!context.isRecover()) {
-      walManager.write(context);
-    }
+  public void preProcess(
+      MemTable root, DeletionRequest deletionRequest, DeleteRequestContext context)
+      throws Exception {
+    walManager.write(deletionRequest);
   }
 
   /** set the delete operation for each layer of memory nodes */
   private void initLevelProcess() {
-    LevelProcessChain levelProcessChain = new LevelProcessChain<>();
+    LevelProcessChain levelProcessChain =
+        new LevelProcessChain<>();
     levelProcessChain
         .nextLevel(new MemTableDeletion())
         .nextLevel(new MemChunkGroupDeletion())
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
index 6b11f648710cb..8996f1e3f0ad0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
@@ -25,7 +26,7 @@
 import java.util.List;
 
 /** deletion for MemChunk */
-public class MemChunkDeletion extends DeleteLevelProcess {
+public class MemChunkDeletion extends DeleteLevelProcess {
 
   /**
    * MemChunk is the last layer of memory nodes, no children
@@ -35,7 +36,8 @@ public class MemChunkDeletion extends DeleteLevelProcess {
    * @return null
    */
   @Override
-  public List getChildren(MemChunk memNode, DeleteRequestContext context) {
+  public List getChildren(
+      MemChunk memNode, DeletionRequest request, DeleteRequestContext context) {
     return null;
   }
 
@@ -46,8 +48,8 @@ public List getChildren(MemChunk memNode, DeleteRequestContext context)
    * @param context deletion request context
    */
   @Override
-  public void delete(MemChunk memNode, DeleteRequestContext context) {
-    Integer deviceID = (Integer) context.getValue();
+  public void delete(MemChunk memNode, DeletionRequest request, DeleteRequestContext context) {
+    Integer deviceID = request.getValue();
     memNode.remove(deviceID);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
index a0f40a069aa93..8dd06a208ba56 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
@@ -27,7 +28,8 @@
 import java.util.List;
 
 /** deletion for MemChunkGroup */
-public class MemChunkGroupDeletion extends DeleteLevelProcess {
+public class MemChunkGroupDeletion
+    extends DeleteLevelProcess {
 
   /**
    * get all MemChunks that need to be processed in the current MemChunkGroup
@@ -37,9 +39,10 @@ public class MemChunkGroupDeletion extends DeleteLevelProcess getChildren(MemChunkGroup memNode, DeleteRequestContext context) {
+  public List getChildren(
+      MemChunkGroup memNode, DeletionRequest deletionRequest, DeleteRequestContext context) {
     List memChunks = new ArrayList<>();
-    String tagValue = (String) context.getKey();
+    String tagValue = deletionRequest.getKey(context);
     MemChunk child = memNode.get(tagValue);
     if (child != null) memChunks.add(child);
     return memChunks;
@@ -52,8 +55,9 @@ public List getChildren(MemChunkGroup memNode, DeleteRequestContext co
    * @param context deletion request context
    */
   @Override
-  public void delete(MemChunkGroup memNode, DeleteRequestContext context) {
-    String tagValue = (String) context.getKey();
+  public void delete(
+      MemChunkGroup memNode, DeletionRequest deletionRequest, DeleteRequestContext context) {
+    String tagValue = deletionRequest.getKey(context);
     MemChunk child = memNode.get(tagValue);
     if (child == null || child.isEmpty()) {
       memNode.remove(tagValue);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
index 4795585b32e57..eff0c21b8c439 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
@@ -28,7 +29,7 @@
 import java.util.Set;
 
 /** deletion for MemTable */
-public class MemTableDeletion extends DeleteLevelProcess {
+public class MemTableDeletion extends DeleteLevelProcess {
 
   /**
    * get all MemChunkGroups that need to be processed in the current MemTable
@@ -38,10 +39,11 @@ public class MemTableDeletion extends DeleteLevelProcess getChildren(MemTable memNode, DeleteRequestContext context) {
+  public List getChildren(
+      MemTable memNode, DeletionRequest deletionRequest, DeleteRequestContext context) {
     if (memNode.isImmutable()) return new ArrayList<>();
     List memChunkGroups = new ArrayList<>();
-    String tagKey = (String) context.getKey();
+    String tagKey = deletionRequest.getKey(context);
     MemChunkGroup child = memNode.get(tagKey);
     if (child != null) memChunkGroups.add(child);
     return memChunkGroups;
@@ -54,15 +56,16 @@ public List getChildren(MemTable memNode, DeleteRequestContext co
    * @param context deletion request context
    */
   @Override
-  public void delete(MemTable memNode, DeleteRequestContext context) {
+  public void delete(
+      MemTable memNode, DeletionRequest deletionRequest, DeleteRequestContext context) {
     if (memNode.isImmutable()) {
       Set deletionList = memNode.getDeletionList();
-      if (!deletionList.contains(context.getValue())) {
-        deletionList.add((Integer) context.getValue());
+      if (!deletionList.contains(deletionRequest.getValue())) {
+        deletionList.add(deletionRequest.getValue());
       }
       return;
     }
-    String tagKey = (String) context.getKey();
+    String tagKey = deletionRequest.getKey(context);
     MemChunkGroup child = memNode.get(tagKey);
     if (child == null || child.isEmpty()) {
       memNode.remove(tagKey);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
index 3ad52edaaec47..aa87c6168c655 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
@@ -27,7 +28,8 @@
 import java.io.IOException;
 
 /** manage insertion to MemTable */
-public class InsertionManager extends BasicLSMManager {
+public class InsertionManager
+    extends BasicLSMManager {
 
   // use wal manager object to write wal file on insertion
   private WALManager walManager;
@@ -45,15 +47,16 @@ public InsertionManager(WALManager walManager) {
    * @throws Exception
    */
   @Override
-  public void preProcess(MemTable root, InsertRequestContext context) throws IOException {
-    if (!context.isRecover()) {
-      walManager.write(context);
-    }
+  public void preProcess(
+      MemTable root, InsertionRequest insertionRequest, InsertRequestContext context)
+      throws IOException {
+    walManager.write(insertionRequest);
   }
 
   /** set the insert operation for each layer of memory nodes */
   private void initLevelProcess() {
-    LevelProcessChain levelProcessChain = new LevelProcessChain<>();
+    LevelProcessChain levelProcessChain =
+        new LevelProcessChain<>();
     levelProcessChain
         .nextLevel(new MemTableInsertion())
         .nextLevel(new MemChunkGroupInsertion())
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
index 5877b53f3d46b..a07f196324bd2 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
@@ -27,7 +28,8 @@
 import java.util.List;
 
 /** insertion for MemChunkGroup */
-public class MemChunkGroupInsertion extends InsertLevelProcess {
+public class MemChunkGroupInsertion
+    extends InsertLevelProcess {
 
   /**
    * get all MemChunks that need to be processed in the current MemChunkGroup
@@ -37,9 +39,10 @@ public class MemChunkGroupInsertion extends InsertLevelProcess getChildren(MemChunkGroup memNode, InsertRequestContext context) {
+  public List getChildren(
+      MemChunkGroup memNode, InsertionRequest insertionRequest, InsertRequestContext context) {
     List memChunks = new ArrayList<>();
-    String tagValue = (String) context.getKey();
+    String tagValue = insertionRequest.getKey(context);
     MemChunk child = memNode.get(tagValue);
     if (child != null) memChunks.add(child);
     return memChunks;
@@ -52,8 +55,9 @@ public List getChildren(MemChunkGroup memNode, InsertRequestContext co
    * @param context insert request context
    */
   @Override
-  public void insert(MemChunkGroup memNode, InsertRequestContext context) {
-    String tagValue = (String) context.getKey();
+  public void insert(
+      MemChunkGroup memNode, InsertionRequest insertionRequest, InsertRequestContext context) {
+    String tagValue = insertionRequest.getKey(context);
     memNode.put(tagValue);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
index 1cf23bd2e1c04..9db253d790d5a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
@@ -25,7 +26,7 @@
 import java.util.List;
 
 /** insertion for MemChunk */
-public class MemChunkInsertion extends InsertLevelProcess {
+public class MemChunkInsertion extends InsertLevelProcess {
 
   /**
    * MemChunk is the last layer of memory nodes, no children
@@ -35,7 +36,8 @@ public class MemChunkInsertion extends InsertLevelProcess {
    * @return null
    */
   @Override
-  public List getChildren(MemChunk memNode, InsertRequestContext context) {
+  public List getChildren(
+      MemChunk memNode, InsertionRequest insertionRequest, InsertRequestContext context) {
     return null;
   }
 
@@ -46,8 +48,9 @@ public List getChildren(MemChunk memNode, InsertRequestContext context)
    * @param context insert request context
    */
   @Override
-  public void insert(MemChunk memNode, InsertRequestContext context) {
-    Integer deviceID = (Integer) context.getValue();
+  public void insert(
+      MemChunk memNode, InsertionRequest insertionRequest, InsertRequestContext context) {
+    Integer deviceID = insertionRequest.getValue();
     memNode.put(deviceID);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
index 0d098e88999ab..7a35b6d65763a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
@@ -27,7 +28,8 @@
 import java.util.List;
 
 /** insertion for MemTable */
-public class MemTableInsertion extends InsertLevelProcess {
+public class MemTableInsertion
+    extends InsertLevelProcess {
 
   /**
    * get all MemChunkGroups that need to be processed in the current MemTable
@@ -37,10 +39,11 @@ public class MemTableInsertion extends InsertLevelProcess getChildren(MemTable memNode, InsertRequestContext context) {
+  public List getChildren(
+      MemTable memNode, InsertionRequest insertionRequest, InsertRequestContext context) {
     if (memNode.isImmutable()) return new ArrayList<>();
     List memChunkGroups = new ArrayList<>();
-    String tagKey = (String) context.getKey();
+    String tagKey = insertionRequest.getKey(context);
     MemChunkGroup child = memNode.get(tagKey);
     if (child != null) memChunkGroups.add(child);
     return memChunkGroups;
@@ -53,9 +56,10 @@ public List getChildren(MemTable memNode, InsertRequestContext co
    * @param context insert request context
    */
   @Override
-  public void insert(MemTable memNode, InsertRequestContext context) {
+  public void insert(
+      MemTable memNode, InsertionRequest insertionRequest, InsertRequestContext context) {
     if (memNode.isImmutable()) return;
-    String tagKey = (String) context.getKey();
+    String tagKey = insertionRequest.getKey(context);
     memNode.put(tagKey);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
index ea312468e028a..51bf85c30379e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
@@ -27,7 +28,7 @@
 import java.util.List;
 
 /** query for MemChunkGroup */
-public class MemChunkGroupQuery extends QueryLevelProcess {
+public class MemChunkGroupQuery extends QueryLevelProcess {
 
   /**
    * get all MemChunks that need to be processed in the current MemChunkGroup
@@ -37,9 +38,10 @@ public class MemChunkGroupQuery extends QueryLevelProcess getChildren(MemChunkGroup memNode, QueryRequestContext context) {
+  public List getChildren(
+      MemChunkGroup memNode, QueryRequest queryRequest, QueryRequestContext context) {
     List memChunks = new ArrayList<>();
-    String tagValue = (String) context.getKey();
+    String tagValue = queryRequest.getKey(context);
     MemChunk child = memNode.get(tagValue);
     if (child != null) memChunks.add(child);
     return memChunks;
@@ -52,5 +54,6 @@ public List getChildren(MemChunkGroup memNode, QueryRequestContext con
    * @param context query request context
    */
   @Override
-  public void query(MemChunkGroup memNode, QueryRequestContext context) {}
+  public void query(
+      MemChunkGroup memNode, QueryRequest queryRequest, QueryRequestContext context) {}
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
index d60be13c8431c..f94ea6d8f43b0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
@@ -27,7 +28,7 @@
 import java.util.List;
 
 /** query for MemChunk */
-public class MemChunkQuery extends QueryLevelProcess {
+public class MemChunkQuery extends QueryLevelProcess {
 
   /**
    * MemChunk is the last layer of memory nodes, no children
@@ -37,7 +38,8 @@ public class MemChunkQuery extends QueryLevelProcess {
    * @return null
    */
   @Override
-  public List getChildren(MemChunk memNode, QueryRequestContext context) {
+  public List getChildren(
+      MemChunk memNode, QueryRequest queryRequest, QueryRequestContext context) {
     return null;
   }
 
@@ -48,10 +50,10 @@ public List getChildren(MemChunk memNode, QueryRequestContext context) {
    * @param context query request context
    */
   @Override
-  public void query(MemChunk memNode, QueryRequestContext context) {
-    RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult();
+  public void query(MemChunk memNode, QueryRequest queryRequest, QueryRequestContext context) {
+    RoaringBitmap roaringBitmap = queryRequest.getResult();
     if (roaringBitmap == null) roaringBitmap = new RoaringBitmap();
     RoaringBitmap now = RoaringBitmap.or(roaringBitmap, memNode.getRoaringBitmap());
-    context.setResult(now);
+    queryRequest.setResult(now);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
index e5ece3a13a99f..3ae18f148af14 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
@@ -30,7 +31,7 @@
 import java.util.Set;
 
 /** query for MemTable */
-public class MemTableQuery extends QueryLevelProcess {
+public class MemTableQuery extends QueryLevelProcess {
 
   /**
    * get all MemChunkGroups that need to be processed in the current MemTable
@@ -40,9 +41,10 @@ public class MemTableQuery extends QueryLevelProcess {
    * @return A list of saved MemChunkGroups
    */
   @Override
-  public List getChildren(MemTable memNode, QueryRequestContext context) {
+  public List getChildren(
+      MemTable memNode, QueryRequest queryRequest, QueryRequestContext context) {
     List memChunkGroups = new ArrayList<>();
-    String tagKey = (String) context.getKey();
+    String tagKey = queryRequest.getKey(context);
     MemChunkGroup child = memNode.get(tagKey);
     if (child != null) memChunkGroups.add(child);
     return memChunkGroups;
@@ -55,10 +57,10 @@ public List getChildren(MemTable memNode, QueryRequestContext con
    * @param context query request context
    */
   @Override
-  public void query(MemTable memNode, QueryRequestContext context) {
+  public void query(MemTable memNode, QueryRequest queryRequest, QueryRequestContext context) {
     // if the memTable is immutable, we need to delete the id in deletionList in the query result
     if (memNode.isImmutable()) {
-      RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult();
+      RoaringBitmap roaringBitmap = queryRequest.getResult();
       Set deletionList = memNode.getDeletionList();
       for (Integer id : deletionList) {
         roaringBitmap.remove(id);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
index 38022663fa565..94f79237ec650 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
@@ -18,13 +18,14 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query;
 
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
 import org.apache.iotdb.lsm.manager.BasicLSMManager;
 
 /** manage insertion to MemTable */
-public class QueryManager extends BasicLSMManager {
+public class QueryManager extends BasicLSMManager {
 
   public QueryManager() {
     initLevelProcess();
@@ -32,7 +33,8 @@ public QueryManager() {
 
   /** set the query operation for each layer of memory nodes */
   private void initLevelProcess() {
-    LevelProcessChain levelProcessChain = new LevelProcessChain<>();
+    LevelProcessChain levelProcessChain =
+        new LevelProcessChain<>();
     levelProcessChain
         .nextLevel(new MemTableQuery())
         .nextLevel(new MemChunkGroupQuery())
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java
index 5c697e1c77b70..cd3bda3d43752 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java
@@ -19,10 +19,10 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover;
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.ITagInvertedIndex;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.request.Request;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -35,6 +35,7 @@ public class RecoverManager {
 
   public RecoverManager(WALManager walManager) {
     this.walManager = walManager;
+    walManager.setRecover(true);
   }
 
   /**
@@ -45,15 +46,20 @@ public RecoverManager(WALManager walManager) {
   public void recover(ITagInvertedIndex tagInvertedIndex) {
     logger.info("recover tagInvertedIndex");
     while (true) {
-      RequestContext context = walManager.read();
-      switch (context.getType()) {
+      Request request = walManager.read();
+      if (request == null) {
+        walManager.setRecover(false);
+        return;
+      }
+      switch (request.getRequestType()) {
         case INSERT:
-          tagInvertedIndex.addTags((InsertRequestContext) context);
+          tagInvertedIndex.addTags((InsertionRequest) request);
           break;
         case DELETE:
-          tagInvertedIndex.removeTags((DeleteRequestContext) context);
+          tagInvertedIndex.removeTags((DeletionRequest) request);
           break;
         default:
+          walManager.setRecover(false);
           return;
       }
     }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
index 2c59777b7dd28..340e9d45ee2ce 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
@@ -21,16 +21,14 @@
 import org.apache.iotdb.commons.utils.TestOnly;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
+import org.apache.iotdb.lsm.request.Request;
 import org.apache.iotdb.lsm.wal.WALReader;
 import org.apache.iotdb.lsm.wal.WALWriter;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 
 /** Manage wal entry writes and reads */
 public class WALManager {
@@ -54,12 +52,15 @@ public class WALManager {
   // directly use the wal reader that comes with the lsm framework
   private WALReader walReader;
 
+  private boolean recover;
+
   public WALManager(String schemaDirPath) throws IOException {
     this.schemaDirPath = schemaDirPath;
     initFile(schemaDirPath);
     int walBufferSize = tagSchemaConfig.getWalBufferSize();
     walWriter = new WALWriter(walFile, walBufferSize, false);
     walReader = new WALReader(walFile, new WALEntry());
+    recover = false;
   }
 
   private void initFile(String schemaDirPath) throws IOException {
@@ -74,16 +75,17 @@ private void initFile(String schemaDirPath) throws IOException {
   /**
    * handle wal log writes for each request context
    *
-   * @param context request context
+   * @param request request context
    * @throws IOException
    */
-  public synchronized void write(RequestContext context) throws IOException {
-    switch (context.getType()) {
+  public synchronized void write(Request request) throws IOException {
+    if (isRecover()) return;
+    switch (request.getRequestType()) {
       case INSERT:
-        process((InsertRequestContext) context);
+        process((InsertionRequest) request);
         break;
       case DELETE:
-        process((DeleteRequestContext) context);
+        process((DeletionRequest) request);
         break;
       default:
         break;
@@ -95,17 +97,17 @@ public synchronized void write(RequestContext context) throws IOException {
    *
    * @return request context
    */
-  public synchronized RequestContext read() {
+  public synchronized Request read() {
     if (walReader.hasNext()) {
       WALEntry walEntry = (WALEntry) walReader.next();
       if (walEntry.getType() == INSERT) {
-        return generateInsertContext(walEntry);
+        return generateInsertRequest(walEntry);
       }
       if (walEntry.getType() == DELETE) {
         return generateDeleteContext(walEntry);
       }
     }
-    return new RequestContext();
+    return null;
   }
 
   /**
@@ -114,14 +116,8 @@ public synchronized RequestContext read() {
    * @param walEntry wal entry
    * @return insert context
    */
-  private InsertRequestContext generateInsertContext(WALEntry walEntry) {
-    InsertRequestContext insertContext = new InsertRequestContext();
-    List objects = new ArrayList<>();
-    objects.addAll(walEntry.getKeys());
-    insertContext.setKeys(objects);
-    insertContext.setValue(walEntry.getDeviceID());
-    insertContext.setRecover(true);
-    return insertContext;
+  private InsertionRequest generateInsertRequest(WALEntry walEntry) {
+    return new InsertionRequest(walEntry.getKeys(), walEntry.getDeviceID());
   }
 
   /**
@@ -130,46 +126,29 @@ private InsertRequestContext generateInsertContext(WALEntry walEntry) {
    * @param walEntry wal entry
    * @return delete context
    */
-  private DeleteRequestContext generateDeleteContext(WALEntry walEntry) {
-    DeleteRequestContext deleteContext =
-        new DeleteRequestContext(walEntry.getDeviceID(), walEntry.getKeys());
-    List objects = new ArrayList<>();
-    objects.addAll(walEntry.getKeys());
-    deleteContext.setKeys(objects);
-    deleteContext.setValue(walEntry.getDeviceID());
-    deleteContext.setRecover(true);
-    return deleteContext;
+  private DeletionRequest generateDeleteContext(WALEntry walEntry) {
+    return new DeletionRequest(walEntry.getKeys(), walEntry.getDeviceID());
   }
 
   /**
    * handle wal log writes for each insert context
    *
-   * @param insertContext insert context
+   * @param request insert request
    * @throws IOException
    */
-  private void process(InsertRequestContext insertContext) throws IOException {
-    List objects = insertContext.getKeys();
-    List keys = new ArrayList<>();
-    for (Object o : objects) {
-      keys.add((String) o);
-    }
-    WALEntry walEntry = new WALEntry(INSERT, keys, (Integer) insertContext.getValue());
+  private void process(InsertionRequest request) throws IOException {
+    WALEntry walEntry = new WALEntry(INSERT, request.getKeys(), request.getValue());
     walWriter.write(walEntry);
   }
 
   /**
    * handle wal log writes for each delete context
    *
-   * @param deleteContext delete context
+   * @param request delete context
    * @throws IOException
    */
-  private void process(DeleteRequestContext deleteContext) throws IOException {
-    List objects = deleteContext.getKeys();
-    List keys = new ArrayList<>();
-    for (Object o : objects) {
-      keys.add((String) o);
-    }
-    WALEntry walEntry = new WALEntry(DELETE, keys, (Integer) deleteContext.getValue());
+  private void process(DeletionRequest request) throws IOException {
+    WALEntry walEntry = new WALEntry(DELETE, request.getKeys(), request.getValue());
     walWriter.write(walEntry);
   }
 
@@ -178,4 +157,12 @@ public void close() throws IOException {
     walWriter.close();
     walReader.close();
   }
+
+  public boolean isRecover() {
+    return recover;
+  }
+
+  public void setRecover(boolean recover) {
+    this.recover = recover;
+  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java
index 548874625265d..1386d0d3c01f7 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java
@@ -20,60 +20,15 @@
 
 import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
 /**
  * represents the context of a deletion request, this class can be extended to implement a custom
  * context
  */
 public class DeleteRequestContext extends RequestContext {
 
-  // save the key of each level
-  List keys;
-
-  // value to delete
-  Object value;
-
   public DeleteRequestContext() {
     super();
-    type = RequestType.DELETE;
-    // post-order traversal strategy is used by default
-    accessStrategy = new PostOrderAccessStrategy();
-  }
-
-  public DeleteRequestContext(Object value, Object... ks) {
-    super();
-    this.value = value;
-    keys = new ArrayList<>();
-    keys.addAll(Arrays.asList(ks));
-    type = RequestType.DELETE;
     // post-order traversal strategy is used by default
     accessStrategy = new PostOrderAccessStrategy();
   }
-
-  public void setKeys(List keys) {
-    this.keys = keys;
-  }
-
-  public void setValue(Object value) {
-    this.value = value;
-  }
-
-  public Object getKey() {
-    return keys.get(level);
-  }
-
-  public List getKeys() {
-    return keys;
-  }
-
-  public Object getValue() {
-    return value;
-  }
-
-  public int size() {
-    return keys.size();
-  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java
index d2a9cd8bae10d..4a66ec82e7fb9 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java
@@ -27,7 +27,6 @@
 public class FlushRequestContext extends RequestContext {
   public FlushRequestContext() {
     super();
-    type = RequestType.FLUSH;
     // use the reverse breadth-first traversal strategy to access memory nodes
     accessStrategy = new RBFSAccessStrategy();
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java
index 933f006a58b0e..0555d35ec5dfb 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java
@@ -20,56 +20,15 @@
 
 import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
 /**
  * represents the context of a insertion request, this class can be extended to implement a custom
  * context
  */
 public class InsertRequestContext extends RequestContext {
 
-  // save the key of each level
-  List keys;
-
-  // value to insert
-  Object value;
-
   public InsertRequestContext() {
     super();
-    type = RequestType.INSERT;
     // preorder traversal strategy is used by default
     accessStrategy = new PreOrderAccessStrategy();
   }
-
-  public InsertRequestContext(Object value, Object... keys) {
-    super();
-    this.value = value;
-    this.keys = new ArrayList<>();
-    this.keys.addAll(Arrays.asList(keys));
-    type = RequestType.INSERT;
-    // preorder traversal strategy is used by default
-    accessStrategy = new PreOrderAccessStrategy();
-  }
-
-  public Object getKey() {
-    return keys.get(level);
-  }
-
-  public void setKeys(List keys) {
-    this.keys = keys;
-  }
-
-  public void setValue(Object value) {
-    this.value = value;
-  }
-
-  public List getKeys() {
-    return keys;
-  }
-
-  public Object getValue() {
-    return value;
-  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java
index 8742ac4e2ace8..2f888571f024f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java
@@ -20,33 +20,15 @@
 
 import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
 /**
  * represents the context of a query request, this class can be extended to implement a custom
  * context
  */
 public class QueryRequestContext extends RequestContext {
 
-  // save the key of each level
-  List keys;
-
-  public QueryRequestContext(Object... ks) {
+  public QueryRequestContext() {
     super();
-    keys = new ArrayList<>();
-    keys.addAll(Arrays.asList(ks));
-    type = RequestType.QUERY;
     // post-order traversal strategy is used by default
     accessStrategy = new PostOrderAccessStrategy();
   }
-
-  public Object getKey() {
-    return keys.get(level);
-  }
-
-  public int size() {
-    return keys.size();
-  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
index 4866be0ca1fd9..3adcd9c807d1c 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
@@ -24,9 +24,6 @@
 /** represents the context of a request */
 public class RequestContext {
 
-  // request type
-  RequestType type;
-
   // memory Structure Access Policy
   AccessStrategy accessStrategy;
 
@@ -36,19 +33,11 @@ public class RequestContext {
   // the maximum level of memory nodes that can be processed
   int levelUpperBound;
 
-  // return value after request processing is complete
-  Object result;
-
-  // whether the request context is only used for recovery
-  boolean recover;
-
   public RequestContext() {
     // preorder traversal strategy is used by default
     accessStrategy = new PreOrderAccessStrategy();
-    type = RequestType.NONE;
     level = 0;
     levelUpperBound = Integer.MAX_VALUE;
-    recover = false;
   }
 
   public void setLevel(int level) {
@@ -59,22 +48,6 @@ public int getLevel() {
     return level;
   }
 
-  public RequestType getType() {
-    return type;
-  }
-
-  public void setType(RequestType type) {
-    this.type = type;
-  }
-
-  public Object getResult() {
-    return result;
-  }
-
-  public void setResult(Object result) {
-    this.result = result;
-  }
-
   public AccessStrategy getAccessStrategy() {
     return accessStrategy;
   }
@@ -90,12 +63,4 @@ public int getLevelUpperBound() {
   public void setLevelUpperBound(int levelUpperBound) {
     this.levelUpperBound = levelUpperBound;
   }
-
-  public boolean isRecover() {
-    return recover;
-  }
-
-  public void setRecover(boolean recover) {
-    this.recover = recover;
-  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
index 4e10ba794bdc3..7f06ef5be133c 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
@@ -23,11 +23,11 @@
 import java.util.List;
 
 /** the processing method corresponding to each layer of memory nodes */
-public abstract class BasicLevelProcess
-    implements LevelProcess {
+public abstract class BasicLevelProcess
+    implements LevelProcess {
 
   // the next level process
-  LevelProcess next;
+  LevelProcess next;
 
   /**
    * process the current layer memory node
@@ -35,7 +35,7 @@ public abstract class BasicLevelProcess
    * @param memNode memory node
    * @param context request context
    */
-  public abstract void handle(I memNode, C context);
+  public abstract void handle(I memNode, R request, C context);
 
   /**
    * get the memory node that needs to be processed in the next layer
@@ -44,7 +44,7 @@ public abstract class BasicLevelProcess
    * @param context request context
    * @return all next-level memory nodes that need to be processed
    */
-  public abstract List getChildren(I memNode, C context);
+  public abstract List getChildren(I memNode, R request, C context);
 
   /**
    * add the LevelProcess of the next layer of memory nodes
@@ -53,7 +53,7 @@ public abstract class BasicLevelProcess
    * @return the next level process
    */
   @Override
-  public  LevelProcess nextLevel(LevelProcess next) {
+  public  LevelProcess nextLevel(LevelProcess next) {
     this.next = next;
     return next;
   }
@@ -65,15 +65,15 @@ public  LevelProcess nextLevel(LevelProcess next) {
    * @param context request context
    */
   @Override
-  public void process(I memNode, C context) {
-    context.getAccessStrategy().execute(this, memNode, context);
+  public void process(I memNode, R request, C context) {
+    context.getAccessStrategy().execute(this, memNode, request, context);
   }
 
   public boolean hasNext() {
     return next != null;
   }
 
-  public LevelProcess getNext() {
+  public LevelProcess getNext() {
     return next;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java
index bd71d7fc86792..850c705c77149 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java
@@ -21,8 +21,8 @@
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 
 /** indicates the deletion method of each layer of memory nodes */
-public abstract class DeleteLevelProcess
-    extends BasicLevelProcess {
+public abstract class DeleteLevelProcess
+    extends BasicLevelProcess {
 
   /**
    * the deletion method of memory node
@@ -30,10 +30,10 @@ public abstract class DeleteLevelProcess
    * @param memNode memory node
    * @param context deletion request context
    */
-  public abstract void delete(I memNode, DeleteRequestContext context);
+  public abstract void delete(I memNode, R request, DeleteRequestContext context);
 
   @Override
-  public void handle(I memNode, DeleteRequestContext context) {
-    delete(memNode, context);
+  public void handle(I memNode, R request, DeleteRequestContext context) {
+    delete(memNode, request, context);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java
index 46d72b4b5b50e..8da40d163efbc 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java
@@ -21,7 +21,8 @@
 import org.apache.iotdb.lsm.context.FlushRequestContext;
 
 /** indicates the flush method of each layer of memory nodes */
-public abstract class FlushLevelProcess extends BasicLevelProcess {
+public abstract class FlushLevelProcess
+    extends BasicLevelProcess {
 
   /**
    * the flush method of memory node
@@ -31,7 +32,7 @@ public abstract class FlushLevelProcess extends BasicLevelProcess
-    extends BasicLevelProcess {
+public abstract class InsertLevelProcess
+    extends BasicLevelProcess {
 
   /**
    * the insertion method of memory node
@@ -30,10 +30,10 @@ public abstract class InsertLevelProcess
    * @param memNode memory node
    * @param context insertion request context
    */
-  public abstract void insert(I memNode, InsertRequestContext context);
+  public abstract void insert(I memNode, R request, InsertRequestContext context);
 
   @Override
-  public void handle(I memNode, InsertRequestContext context) {
-    insert(memNode, context);
+  public void handle(I memNode, R request, InsertRequestContext context) {
+    insert(memNode, request, context);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java
index 763d6eb4f6d32..1eb0379906c13 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java
@@ -21,7 +21,7 @@
 import org.apache.iotdb.lsm.context.RequestContext;
 
 /** the processing method corresponding to each layer of memory nodes */
-public interface LevelProcess {
+public interface LevelProcess {
 
   /**
    * add the LevelProcess of the next layer of memory nodes
@@ -29,7 +29,7 @@ public interface LevelProcess {
    * @param next LevelProcess of the next layer
    * @return LevelProcess of the next layer
    */
-   LevelProcess nextLevel(LevelProcess next);
+   LevelProcess nextLevel(LevelProcess next);
 
   /**
    * use this method to process memory nodes at each layer according to the access strategy
@@ -37,5 +37,5 @@ public interface LevelProcess {
    * @param memNode memory node
    * @param context request context
    */
-  void process(I memNode, C context);
+  void process(I memNode, R request, C context);
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
index deeaff99b4b3e..9f34eac6413a8 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
@@ -20,17 +20,17 @@
 
 import org.apache.iotdb.lsm.context.RequestContext;
 
-public class LevelProcessChain {
+public class LevelProcessChain {
 
   // the level process of the first layer of memory nodes
-  LevelProcess headLevelProcess;
+  LevelProcess headLevelProcess;
 
-  public  LevelProcess nextLevel(LevelProcess next) {
+  public  LevelProcess nextLevel(LevelProcess next) {
     this.headLevelProcess = next;
     return next;
   }
 
-  public void process(T memNode, C context) {
-    headLevelProcess.process(memNode, context);
+  public void process(T memNode, R request, C context) {
+    headLevelProcess.process(memNode, request, context);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java
index 2fef2f213d058..93f1c481f4d1b 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java
@@ -21,7 +21,8 @@
 import org.apache.iotdb.lsm.context.QueryRequestContext;
 
 /** indicates the query method of each layer of memory nodes */
-public abstract class QueryLevelProcess extends BasicLevelProcess {
+public abstract class QueryLevelProcess
+    extends BasicLevelProcess {
 
   /**
    * the query method of memory node
@@ -29,10 +30,10 @@ public abstract class QueryLevelProcess extends BasicLevelProcess implements LSMManager {
+public class BasicLSMManager implements LSMManager {
 
   // the level process of the first layer of memory nodes
-  LevelProcessChain levelProcessChain;
+  LevelProcessChain levelProcessChain;
 
   public BasicLSMManager() {}
 
-  public BasicLSMManager(LevelProcessChain levelProcessChain) {
+  public BasicLSMManager(LevelProcessChain levelProcessChain) {
     this.levelProcessChain = levelProcessChain;
   }
 
@@ -41,7 +41,7 @@ public BasicLSMManager(LevelProcessChain levelProcessChain) {
    * @throws Exception
    */
   @Override
-  public void preProcess(T root, C context) throws Exception {}
+  public void preProcess(T root, R request, C context) throws Exception {}
 
   /**
    * postprocessing of the root memory node
@@ -51,7 +51,7 @@ public void preProcess(T root, C context) throws Exception {}
    * @throws Exception
    */
   @Override
-  public void postProcess(T root, C context) throws Exception {}
+  public void postProcess(T root, R request, C context) throws Exception {}
 
   /**
    * processing of the root memory node
@@ -61,13 +61,13 @@ public void postProcess(T root, C context) throws Exception {}
    * @throws Exception
    */
   @Override
-  public void process(T root, C context) throws Exception {
-    preProcess(root, context);
-    levelProcessChain.process(root, context);
-    postProcess(root, context);
+  public void process(T root, R request, C context) throws Exception {
+    preProcess(root, request, context);
+    levelProcessChain.process(root, request, context);
+    postProcess(root, request, context);
   }
 
-  public void setLevelProcessChain(LevelProcessChain levelProcessChain) {
+  public void setLevelProcessChain(LevelProcessChain levelProcessChain) {
     this.levelProcessChain = levelProcessChain;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java
index c5d4169a75dd2..3783f7f14c513 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java
@@ -21,7 +21,7 @@
 import org.apache.iotdb.lsm.context.RequestContext;
 
 // used to implement lsm manager
-public interface LSMManager {
+public interface LSMManager {
 
   /**
    * preprocessing of the root memory node
@@ -30,7 +30,7 @@ public interface LSMManager {
    * @param context request context
    * @throws Exception
    */
-  void preProcess(T root, C context) throws Exception;
+  void preProcess(T root, R request, C context) throws Exception;
 
   /**
    * postprocessing of the root memory node
@@ -39,7 +39,7 @@ public interface LSMManager {
    * @param context request context
    * @throws Exception
    */
-  void postProcess(T root, C context) throws Exception;
+  void postProcess(T root, R request, C context) throws Exception;
 
   /**
    * use this method to process root memory node
@@ -47,5 +47,5 @@ public interface LSMManager {
    * @param memNode memory node
    * @param context request context
    */
-  void process(T memNode, C context) throws Exception;
+  void process(T memNode, R request, C context) throws Exception;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java
new file mode 100644
index 0000000000000..36f2ed69b112c
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.request;
+
+public abstract class DeletionRequest extends Request {
+
+  public DeletionRequest() {
+    requestType = RequestType.DELETE;
+  }
+
+  public abstract V getValue();
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java
new file mode 100644
index 0000000000000..80e32626ac339
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.request;
+
+public abstract class InsertionRequest extends Request {
+
+  public InsertionRequest() {
+    requestType = RequestType.INSERT;
+  }
+
+  public abstract V getValue();
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java
new file mode 100644
index 0000000000000..9ad69df9d067d
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.request;
+
+public abstract class QueryRequest extends Request {
+
+  public QueryRequest() {
+    requestType = RequestType.QUERY;
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java
new file mode 100644
index 0000000000000..dd6fcef56ab1a
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.request;
+
+import org.apache.iotdb.lsm.context.RequestContext;
+
+import java.util.List;
+
+public abstract class Request {
+
+  RequestType requestType;
+
+  public Request() {
+    requestType = RequestType.NONE;
+  }
+
+  public abstract K getKey(RequestContext context);
+
+  public abstract R getResult();
+
+  public abstract void setResult(R result);
+
+  public abstract List getKeys();
+
+  public RequestType getRequestType() {
+    return requestType;
+  }
+
+  public void setRequestType(RequestType requestType) {
+    this.requestType = requestType;
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestType.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java
similarity index 95%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestType.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java
index e7dfac6c2a9e9..e88d3104eafb6 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestType.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.lsm.context;
+package org.apache.iotdb.lsm.request;
 
 public enum RequestType {
   NONE,
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java
index 567f83faa3e3b..bf45715f0482f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java
@@ -31,6 +31,6 @@ public interface AccessStrategy {
    * @param memNode memory node
    * @param context request context
    */
-   void execute(
-      BasicLevelProcess levelProcess, I memNode, C context);
+   void execute(
+      BasicLevelProcess levelProcess, I memNode, R request, C context);
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
index e93bd5ca959e6..28075beb0d6a7 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
@@ -40,27 +40,27 @@ public class BFSAccessStrategy implements AccessStrategy {
    * @param context request context
    */
   @Override
-  public  void execute(
-      BasicLevelProcess levelProcess, I memNode, C context) {
+  public  void execute(
+      BasicLevelProcess levelProcess, I memNode, R request, C context) {
     List children = new ArrayList<>();
     int currentLevel = context.getLevel();
     if (sameLevelMemNodes == null) {
       sameLevelMemNodes = new LinkedList<>();
       // process the current memory node
-      levelProcess.handle(memNode, context);
+      levelProcess.handle(memNode, request, context);
       // get all memory nodes to be processed in the next layer
-      children = levelProcess.getChildren(memNode, context);
+      children = levelProcess.getChildren(memNode, request, context);
     } else {
       while (!sameLevelMemNodes.isEmpty()) {
         I node = (I) sameLevelMemNodes.poll();
-        levelProcess.handle(node, context);
-        children.addAll(levelProcess.getChildren(node, context));
+        levelProcess.handle(node, request, context);
+        children.addAll(levelProcess.getChildren(node, request, context));
       }
     }
     sameLevelMemNodes.addAll(children);
     context.setLevel(currentLevel + 1);
     if (levelProcess.hasNext() && !sameLevelMemNodes.isEmpty()) {
-      levelProcess.getNext().process(null, context);
+      levelProcess.getNext().process(null, request, context);
     }
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
index 5e96f29b1b451..86986f41c5c6b 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
@@ -34,23 +34,23 @@ public class PostOrderAccessStrategy implements AccessStrategy {
    * @param context request context
    */
   @Override
-  public  void execute(
-      BasicLevelProcess levelProcess, I memNode, C context) {
+  public  void execute(
+      BasicLevelProcess levelProcess, I memNode, R request, C context) {
     int currentLevel = context.getLevel();
     AccessStrategy accessStrategy = context.getAccessStrategy();
     // get all memory nodes to be processed in the next layer
-    List children = levelProcess.getChildren(memNode, context);
+    List children = levelProcess.getChildren(memNode, request, context);
     if (levelProcess.hasNext()) {
       context.setLevel(currentLevel + 1);
       for (O child : children) {
         // process next level memory node
-        levelProcess.getNext().process(child, context);
+        levelProcess.getNext().process(child, request, context);
       }
     }
 
     context.setLevel(currentLevel);
     context.setAccessStrategy(accessStrategy);
     // process the current memory node
-    levelProcess.handle(memNode, context);
+    levelProcess.handle(memNode, request, context);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
index d16780cf59e58..cb423d5b44a7d 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
@@ -34,19 +34,19 @@ public class PreOrderAccessStrategy implements AccessStrategy {
    * @param context request context
    */
   @Override
-  public  void execute(
-      BasicLevelProcess levelProcess, I memNode, C context) {
+  public  void execute(
+      BasicLevelProcess levelProcess, I memNode, R request, C context) {
     int currentLevel = context.getLevel();
     // process the current memory node
-    levelProcess.handle(memNode, context);
+    levelProcess.handle(memNode, request, context);
     // get all memory nodes to be processed in the next layer
-    List children = levelProcess.getChildren(memNode, context);
+    List children = levelProcess.getChildren(memNode, request, context);
 
     if (levelProcess.hasNext()) {
       context.setLevel(currentLevel + 1);
       for (O child : children) {
         // process next level memory node
-        levelProcess.getNext().process(child, context);
+        levelProcess.getNext().process(child, request, context);
       }
     }
     context.setLevel(currentLevel);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
index 13b05c51717ae..b6b0c14a28a89 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
@@ -34,8 +34,8 @@ public class RBFSAccessStrategy implements AccessStrategy {
    * @param context request context
    */
   @Override
-  public  void execute(
-      BasicLevelProcess levelProcess, I memNode, C context) {
+  public  void execute(
+      BasicLevelProcess levelProcess, I memNode, R request, C context) {
     int currentLevel = context.getLevel();
 
     // if the upper bound has not been set and there is no next-level processing method, set the
@@ -49,11 +49,11 @@ public  void execute(
       // if all the next level nodes of the root node have not been processed
       while (context.getLevelUpperBound() != currentLevel) {
         // process all pending next-level nodes
-        List children = levelProcess.getChildren(memNode, context);
+        List children = levelProcess.getChildren(memNode, request, context);
         for (O child : children) {
           context.setLevel(currentLevel + 1);
           // use the processing method of the next layer to process the next layer of nodes
-          levelProcess.getNext().process(child, context);
+          levelProcess.getNext().process(child, request, context);
           context.setLevel(currentLevel);
         }
 
@@ -62,7 +62,7 @@ public  void execute(
       }
 
       // process the current memory node
-      levelProcess.handle(memNode, context);
+      levelProcess.handle(memNode, request, context);
       return;
     }
 
@@ -70,15 +70,15 @@ public  void execute(
 
     // only process memory nodes with equal level and upper bound
     if (currentLevel == context.getLevelUpperBound()) {
-      levelProcess.handle(memNode, context);
+      levelProcess.handle(memNode, request, context);
       return;
     }
 
     // process all pending next-level nodes
-    List children = levelProcess.getChildren(memNode, context);
+    List children = levelProcess.getChildren(memNode, request, context);
     for (O child : children) {
       context.setLevel(currentLevel + 1);
-      levelProcess.getNext().process(child, context);
+      levelProcess.getNext().process(child, request, context);
       context.setLevel(currentLevel);
     }
   }
diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java
index 5f4ddb78f34dc..5153de805afe7 100644
--- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java
+++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java
@@ -214,7 +214,7 @@ private void createAlignedTimeseries() throws Exception {
     tagSchemaRegion.createAlignedTimeSeries(plan);
     plan =
         new CreateAlignedTimeSeriesPlan(
-            new PartialPath(storageGroup + "tag1.x.tag2.y"),
+            new PartialPath(storageGroup + ".tag1.x.tag2.y"),
             Arrays.asList("s1", "s2", "s3"),
             Arrays.asList(
                 TSDataType.valueOf("FLOAT"),

From 9555298118ee9c5d6999e0e12772594549ec7a1b Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sun, 23 Oct 2022 14:24:12 +0800
Subject: [PATCH 37/53] LSMEngine and MemTableGroup

---
 .../tagSchemaRegion/TagSchemaRegion.java      |   6 +-
 .../tagIndex/InvertedIndex.java               | 195 ++++++++++++++++++
 .../tagIndex/Request/DeletionRequest.java     |   2 +-
 .../tagIndex/Request/InsertionRequest.java    |   2 +-
 .../tagIndex/Request/QueryRequest.java        |   5 +-
 .../tagIndex/TagInvertedIndex.java            |   2 +-
 .../tagIndex/deletion/DeletionManager.java    |   4 +
 .../deletion/MemTableGroupDeletion.java       |  49 +++++
 .../tagIndex/insertion/InsertionManager.java  |   4 +
 .../insertion/MemTableGroupInsertion.java     |  58 ++++++
 .../tagIndex/memtable/MemTableGroup.java      | 100 +++++++++
 .../tagIndex/query/MemTableGroupQuery.java    |  42 ++++
 .../tagIndex/query/QueryManager.java          |   8 +
 .../tagIndex/wal/WALEntry.java                |   8 +-
 .../tagIndex/wal/WALManager.java              |  73 ++-----
 .../iotdb/lsm/context/RequestContext.java     |   8 +-
 .../apache/iotdb/lsm/engine/ILSMEngine.java   |  52 +++++
 .../apache/iotdb/lsm/engine/LSMEngine.java    | 170 +++++++++++++++
 .../lsm/levelProcess/BasicLevelProcess.java   |   8 +-
 .../{LevelProcess.java => ILevelProcess.java} |   4 +-
 .../lsm/levelProcess/LevelProcessChain.java   |   7 +-
 .../iotdb/lsm/manager/BasicLSMManager.java    |  26 +--
 .../iotdb/lsm/manager/DeletionManager.java    |  49 +++++
 .../{LSMManager.java => ILSMManager.java}     |   2 +-
 .../iotdb/lsm/manager/InsertionManager.java   |  52 +++++
 .../iotdb/lsm/manager/QueryManager.java       |  32 +++
 .../iotdb/lsm/manager/RecoverManager.java     |  49 +++++
 .../apache/iotdb/lsm/manager/WALManager.java  | 126 +++++++++++
 .../iotdb/lsm/recover/IRecoverable.java       |  26 +++
 .../iotdb/lsm/request/DeletionRequest.java    |   2 +-
 .../iotdb/lsm/request/InsertionRequest.java   |   2 +-
 .../iotdb/lsm/request/QueryRequest.java       |   2 +-
 .../org/apache/iotdb/lsm/request/Request.java |   2 +-
 .../iotdb/lsm/strategy/BFSAccessStrategy.java |   2 +-
 ...cessStrategy.java => IAccessStrategy.java} |   2 +-
 .../lsm/strategy/PostOrderAccessStrategy.java |   4 +-
 .../lsm/strategy/PreOrderAccessStrategy.java  |   2 +-
 .../lsm/strategy/RBFSAccessStrategy.java      |   2 +-
 .../org/apache/iotdb/lsm/wal/WALRecord.java   |   7 +-
 .../tagIndex/InvertedIndexTest.java           | 184 +++++++++++++++++
 40 files changed, 1268 insertions(+), 112 deletions(-)
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndex.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/{LevelProcess.java => ILevelProcess.java} (90%)
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/{LSMManager.java => ILSMManager.java} (96%)
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/{AccessStrategy.java => IAccessStrategy.java} (97%)
 create mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndexTest.java

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
index ac0611fbca2cc..0a0ef22541290 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
@@ -55,7 +55,7 @@
 import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion;
 import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.idtable.IDTableWithDeviceIDListImpl;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.InvertedIndex;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.MeasurementPathUtils;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.PathTagConverterUtils;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.ShowTimeSeriesResultUtils;
@@ -110,7 +110,7 @@ public class TagSchemaRegion implements ISchemaRegion {
   private final String schemaRegionDirPath;
 
   // tag inverted index
-  private final TagInvertedIndex tagInvertedIndex;
+  private final InvertedIndex tagInvertedIndex;
 
   // manager timeSeries, and use a deviceID list manager device id -> INT32 id
   private final IDTableWithDeviceIDListImpl idTableWithDeviceIDList;
@@ -131,7 +131,7 @@ public TagSchemaRegion(
     this.seriesNumerLimiter = seriesNumerLimiter;
     File schemaRegionDir = new File(schemaRegionDirPath);
     idTableWithDeviceIDList = new IDTableWithDeviceIDListImpl(schemaRegionDir);
-    tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath);
+    tagInvertedIndex = new InvertedIndex(schemaRegionDirPath);
     init();
   }
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndex.java
new file mode 100644
index 0000000000000..f4a8f628de27d
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndex.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
+
+import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkGroupDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableGroupDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkGroupInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableGroupInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkGroupQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableGroupQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
+import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.engine.LSMEngine;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.manager.DeletionManager;
+import org.apache.iotdb.lsm.manager.InsertionManager;
+import org.apache.iotdb.lsm.manager.QueryManager;
+import org.apache.iotdb.lsm.manager.RecoverManager;
+
+import org.roaringbitmap.RoaringBitmap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class InvertedIndex implements ITagInvertedIndex {
+
+  private static final String WAL_FILE_NAME = "tag_inverted_index.log";
+
+  private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class);
+
+  private static final TagSchemaConfig tagSchemaConfig =
+      TagSchemaDescriptor.getInstance().getTagSchemaConfig();
+
+  LSMEngine lsmEngine;
+
+  public InvertedIndex(String schemaDirPath) {
+    LevelProcessChain
+        insertionLevelProcessChain = new LevelProcessChain<>();
+    LevelProcessChain
+        deletionLevelProcessChain = new LevelProcessChain<>();
+    LevelProcessChain queryLevelProcessChain =
+        new LevelProcessChain<>();
+    insertionLevelProcessChain
+        .nextLevel(new MemTableGroupInsertion())
+        .nextLevel(new MemTableInsertion())
+        .nextLevel(new MemChunkGroupInsertion())
+        .nextLevel(new MemChunkInsertion());
+    deletionLevelProcessChain
+        .nextLevel(new MemTableGroupDeletion())
+        .nextLevel(new MemTableDeletion())
+        .nextLevel(new MemChunkGroupDeletion())
+        .nextLevel(new MemChunkDeletion());
+    queryLevelProcessChain
+        .nextLevel(new MemTableGroupQuery())
+        .nextLevel(new MemTableQuery())
+        .nextLevel(new MemChunkGroupQuery())
+        .nextLevel(new MemChunkQuery());
+    try {
+      WALManager walManager =
+          new WALManager(
+              schemaDirPath,
+              WAL_FILE_NAME,
+              tagSchemaConfig.getWalBufferSize(),
+              new WALEntry(),
+              false);
+      InsertionManager insertionManager =
+          new InsertionManager<>(walManager);
+      DeletionManager deletionManager =
+          new DeletionManager<>(walManager);
+      QueryManager queryManager = new QueryManager<>();
+      RecoverManager> recoverManager = new RecoverManager<>(walManager);
+      insertionManager.setLevelProcessChain(insertionLevelProcessChain);
+      deletionManager.setLevelProcessChain(deletionLevelProcessChain);
+      queryManager.setLevelProcessChain(queryLevelProcessChain);
+      lsmEngine = new LSMEngine<>();
+      lsmEngine.setDeletionManager(deletionManager);
+      lsmEngine.setInsertionManager(insertionManager);
+      lsmEngine.setQueryManager(queryManager);
+      lsmEngine.setWalManager(walManager);
+      lsmEngine.setRecoverManager(recoverManager);
+      lsmEngine.setRootMemNode(new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable()));
+      lsmEngine.recover();
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+  }
+
+  @Override
+  public void addTags(InsertionRequest insertionRequest) {}
+
+  @Override
+  public synchronized void addTags(Map tags, int id) {
+    try {
+      for (Map.Entry tag : tags.entrySet()) {
+        InsertionRequest insertionRequest =
+            new InsertionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
+        lsmEngine.insert(insertionRequest);
+      }
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+  }
+
+  @Override
+  public void removeTags(DeletionRequest deletionRequest) {}
+
+  @Override
+  public synchronized void removeTags(Map tags, int id) {
+    try {
+      for (Map.Entry tag : tags.entrySet()) {
+        DeletionRequest deletionRequest =
+            new DeletionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
+        lsmEngine.delete(deletionRequest);
+      }
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+  }
+
+  @Override
+  public synchronized List getMatchedIDs(Map tags) {
+    RoaringBitmap roaringBitmap = new RoaringBitmap();
+    int i = 0;
+    try {
+      for (Map.Entry tag : tags.entrySet()) {
+        RoaringBitmap rb = getMatchedIDs(tag.getKey(), tag.getValue());
+        if (rb == null) continue;
+        else {
+          if (i == 0) roaringBitmap = rb;
+          else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb);
+          i++;
+        }
+      }
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+    return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList());
+  }
+
+  private List generateKeys(String tagKey, String tagValue) {
+    List keys = new ArrayList<>();
+    keys.add(tagKey);
+    keys.add(tagValue);
+    return keys;
+  }
+
+  private RoaringBitmap getMatchedIDs(String tagKey, String tagValue) throws Exception {
+    QueryRequest queryRequest = new QueryRequest(generateKeys(tagKey, tagValue));
+    lsmEngine.query(queryRequest);
+    return queryRequest.getResult();
+  }
+
+  @TestOnly
+  public void clear() throws IOException {
+    lsmEngine.clear();
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
index d35738e6c9157..bc9f24a899039 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
@@ -40,7 +40,7 @@ public DeletionRequest(List keys, int value) {
 
   @Override
   public String getKey(RequestContext context) {
-    return keys.get(context.getLevel());
+    return keys.get(context.getLevel() - 1);
   }
 
   @Override
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
index 52afb9b1a339f..a475f553ece08 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
@@ -44,7 +44,7 @@ public InsertionRequest(List keys, int value) {
 
   @Override
   public String getKey(RequestContext context) {
-    return keys.get(context.getLevel());
+    return keys.get(context.getLevel() - 1);
   }
 
   @Override
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
index 18f816d265bbf..c62d0db758041 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
@@ -19,13 +19,12 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.request.Request;
 
 import org.roaringbitmap.RoaringBitmap;
 
 import java.util.List;
 
-public class QueryRequest extends Request {
+public class QueryRequest extends org.apache.iotdb.lsm.request.QueryRequest {
 
   List keys;
 
@@ -39,7 +38,7 @@ public QueryRequest(List keys) {
 
   @Override
   public String getKey(RequestContext context) {
-    return keys.get(context.getLevel());
+    return keys.get(context.getLevel() - 1);
   }
 
   @Override
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index 939680e50fd2f..2eada5e7042f2 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -87,7 +87,7 @@ public TagInvertedIndex(String schemaDirPath) {
       numOfDeviceIdsInMemTable = tagSchemaConfig.getNumOfDeviceIdsInMemTable();
       maxDeviceID = 0;
       recover();
-    } catch (IOException e) {
+    } catch (Exception e) {
       logger.error("create TagInvertedIndex fail", e);
     }
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
index 7297f46c2a5a1..dad8df35d8730 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
@@ -51,6 +51,10 @@ public void preProcess(
     walManager.write(deletionRequest);
   }
 
+  @Override
+  public void postProcess(MemTable root, DeletionRequest request, DeleteRequestContext context)
+      throws Exception {}
+
   /** set the delete operation for each layer of memory nodes */
   private void initLevelProcess() {
     LevelProcessChain levelProcessChain =
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
new file mode 100644
index 0000000000000..7091f233c1b92
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion;
+
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MemTableGroupDeletion
+    extends DeleteLevelProcess {
+  @Override
+  public List getChildren(
+      MemTableGroup memNode, DeletionRequest request, DeleteRequestContext context) {
+    List memTables = new ArrayList<>();
+    int id = request.getValue();
+    if (memNode.inWorkingMemTable(id)) {
+      memTables.add(memNode.getWorkingMemTable());
+    } else {
+      memTables.add(
+          memNode.getImmutableMemTables().get(id / memNode.getNumOfDeviceIdsInMemTable()));
+    }
+    return memTables;
+  }
+
+  @Override
+  public void delete(
+      MemTableGroup memNode, DeletionRequest request, DeleteRequestContext context) {}
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
index aa87c6168c655..d272e3476abbf 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
@@ -53,6 +53,10 @@ public void preProcess(
     walManager.write(insertionRequest);
   }
 
+  @Override
+  public void postProcess(MemTable root, InsertionRequest request, InsertRequestContext context)
+      throws Exception {}
+
   /** set the insert operation for each layer of memory nodes */
   private void initLevelProcess() {
     LevelProcessChain levelProcessChain =
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
new file mode 100644
index 0000000000000..a4b8fd4adae00
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion;
+
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class MemTableGroupInsertion
+    extends InsertLevelProcess {
+
+  @Override
+  public List getChildren(
+      MemTableGroup memNode, InsertionRequest request, InsertRequestContext context) {
+    List memTables = new ArrayList<>();
+    memTables.add(memNode.getWorkingMemTable());
+    return memTables;
+  }
+
+  @Override
+  public void insert(
+      MemTableGroup memNode, InsertionRequest request, InsertRequestContext context) {
+    int id = request.getValue();
+    MemTable workingMemTable = memNode.getWorkingMemTable();
+    Map immutableMemTables = memNode.getImmutableMemTables();
+    // if the device id can not be saved to the current working MemTable
+    if (!memNode.inWorkingMemTable(id)) {
+      workingMemTable.setStatus(MemTable.IMMUTABLE);
+      immutableMemTables.put(
+          memNode.getMaxDeviceID() / memNode.getNumOfDeviceIdsInMemTable(), workingMemTable);
+      memNode.setWorkingMemTable(new MemTable(MemTable.WORKING));
+    }
+    memNode.setMaxDeviceID(id);
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java
new file mode 100644
index 0000000000000..6df2b43057563
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** used to manage working and immutableMemTable */
+public class MemTableGroup {
+
+  // the maximum number of device ids managed by a working memTable
+  private int numOfDeviceIdsInMemTable;
+
+  // (maxDeviceID / numOfDeviceIdsInMemTable) -> MemTable
+  private Map immutableMemTables;
+
+  private MemTable workingMemTable;
+
+  // the largest device id saved by the current MemTable
+  private int maxDeviceID;
+
+  public MemTableGroup(int numOfDeviceIdsInMemTable) {
+    this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable;
+    workingMemTable = new MemTable(MemTable.WORKING);
+    immutableMemTables = new HashMap<>();
+    maxDeviceID = 0;
+  }
+
+  public int getNumOfDeviceIdsInMemTable() {
+    return numOfDeviceIdsInMemTable;
+  }
+
+  public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) {
+    this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable;
+  }
+
+  public Map getImmutableMemTables() {
+    return immutableMemTables;
+  }
+
+  public void setImmutableMemTables(Map immutableMemTables) {
+    this.immutableMemTables = immutableMemTables;
+  }
+
+  public MemTable getWorkingMemTable() {
+    return workingMemTable;
+  }
+
+  public void setWorkingMemTable(MemTable workingMemTable) {
+    this.workingMemTable = workingMemTable;
+  }
+
+  public int getMaxDeviceID() {
+    return maxDeviceID;
+  }
+
+  public void setMaxDeviceID(int maxDeviceID) {
+    this.maxDeviceID = maxDeviceID;
+  }
+
+  /**
+   * determine whether the id can be saved to the current MemTable
+   *
+   * @param id INT32 device id
+   * @return return true if it can, otherwise return false
+   */
+  public boolean inWorkingMemTable(int id) {
+    return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable;
+  }
+
+  @Override
+  public String toString() {
+    return "MemTableGroup{"
+        + "numOfDeviceIdsInMemTable="
+        + numOfDeviceIdsInMemTable
+        + ", immutableMemTables="
+        + immutableMemTables
+        + ", workingMemTable="
+        + workingMemTable
+        + ", maxDeviceID="
+        + maxDeviceID
+        + '}';
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
new file mode 100644
index 0000000000000..03efb4bca85d1
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query;
+
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MemTableGroupQuery extends QueryLevelProcess {
+  @Override
+  public List getChildren(
+      MemTableGroup memNode, QueryRequest request, QueryRequestContext context) {
+    List memTables = new ArrayList<>();
+    memTables.add(memNode.getWorkingMemTable());
+    memTables.addAll(memNode.getImmutableMemTables().values());
+    return memTables;
+  }
+
+  @Override
+  public void query(MemTableGroup memNode, QueryRequest request, QueryRequestContext context) {}
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
index 94f79237ec650..e5d97dd15e0c3 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
@@ -41,4 +41,12 @@ private void initLevelProcess() {
         .nextLevel(new MemChunkQuery());
     setLevelProcessChain(levelProcessChain);
   }
+
+  @Override
+  public void preProcess(MemTable root, QueryRequest request, QueryRequestContext context)
+      throws Exception {}
+
+  @Override
+  public void postProcess(MemTable root, QueryRequest request, QueryRequestContext context)
+      throws Exception {}
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java
index c2874a8290daf..1b769a2ab7529 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java
@@ -28,7 +28,7 @@
 import java.util.List;
 
 /** represents a record in the wal file */
-public class WALEntry implements WALRecord {
+public class WALEntry implements WALRecord {
 
   // can be insertion(1) or deletion(2)
   private int type;
@@ -105,10 +105,16 @@ public void setType(int type) {
     this.type = type;
   }
 
+  @Override
   public List getKeys() {
     return keys;
   }
 
+  @Override
+  public Integer getValue() {
+    return getDeviceID();
+  }
+
   public void setKeys(List keys) {
     this.keys = keys;
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
index 340e9d45ee2ce..bf79ea903853e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
@@ -18,58 +18,32 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal;
 
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.lsm.request.Request;
 import org.apache.iotdb.lsm.wal.WALReader;
-import org.apache.iotdb.lsm.wal.WALWriter;
+import org.apache.iotdb.lsm.wal.WALRecord;
 
-import java.io.File;
 import java.io.IOException;
 
 /** Manage wal entry writes and reads */
-public class WALManager {
-
-  private static final String WAL_FILE_NAME = "tag_inverted_index.log";
-
+public class WALManager extends org.apache.iotdb.lsm.manager.WALManager {
   private static final int INSERT = 1;
 
   private static final int DELETE = 2;
 
-  private final TagSchemaConfig tagSchemaConfig =
-      TagSchemaDescriptor.getInstance().getTagSchemaConfig();
-
-  private final String schemaDirPath;
-
-  private File walFile;
-
-  // directly use the wal writer that comes with the lsm framework
-  private WALWriter walWriter;
-
-  // directly use the wal reader that comes with the lsm framework
-  private WALReader walReader;
-
-  private boolean recover;
-
-  public WALManager(String schemaDirPath) throws IOException {
-    this.schemaDirPath = schemaDirPath;
-    initFile(schemaDirPath);
-    int walBufferSize = tagSchemaConfig.getWalBufferSize();
-    walWriter = new WALWriter(walFile, walBufferSize, false);
-    walReader = new WALReader(walFile, new WALEntry());
-    recover = false;
+  public WALManager(
+      String schemaDirPath,
+      String walFileName,
+      int walBufferSize,
+      WALRecord walRecord,
+      boolean forceEachWrite)
+      throws IOException {
+    super(schemaDirPath, walFileName, walBufferSize, walRecord, forceEachWrite);
   }
 
-  private void initFile(String schemaDirPath) throws IOException {
-    File schemaDir = new File(schemaDirPath);
-    schemaDir.mkdirs();
-    walFile = new File(this.schemaDirPath, WAL_FILE_NAME);
-    if (!walFile.exists()) {
-      walFile.createNewFile();
-    }
+  public WALManager(String schemaDirPath) {
+    super(schemaDirPath);
   }
 
   /**
@@ -78,6 +52,7 @@ private void initFile(String schemaDirPath) throws IOException {
    * @param request request context
    * @throws IOException
    */
+  @Override
   public synchronized void write(Request request) throws IOException {
     if (isRecover()) return;
     switch (request.getRequestType()) {
@@ -97,9 +72,11 @@ public synchronized void write(Request request) throws IOException {
    *
    * @return request context
    */
+  @Override
   public synchronized Request read() {
+    WALReader walReader = getWalReader();
     if (walReader.hasNext()) {
-      WALEntry walEntry = (WALEntry) walReader.next();
+      WALEntry walEntry = (WALEntry) getWalReader().next();
       if (walEntry.getType() == INSERT) {
         return generateInsertRequest(walEntry);
       }
@@ -138,7 +115,7 @@ private DeletionRequest generateDeleteContext(WALEntry walEntry) {
    */
   private void process(InsertionRequest request) throws IOException {
     WALEntry walEntry = new WALEntry(INSERT, request.getKeys(), request.getValue());
-    walWriter.write(walEntry);
+    getWalWriter().write(walEntry);
   }
 
   /**
@@ -149,20 +126,6 @@ private void process(InsertionRequest request) throws IOException {
    */
   private void process(DeletionRequest request) throws IOException {
     WALEntry walEntry = new WALEntry(DELETE, request.getKeys(), request.getValue());
-    walWriter.write(walEntry);
-  }
-
-  @TestOnly
-  public void close() throws IOException {
-    walWriter.close();
-    walReader.close();
-  }
-
-  public boolean isRecover() {
-    return recover;
-  }
-
-  public void setRecover(boolean recover) {
-    this.recover = recover;
+    getWalWriter().write(walEntry);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
index 3adcd9c807d1c..ee0d3a6f88b72 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
@@ -18,14 +18,14 @@
  */
 package org.apache.iotdb.lsm.context;
 
-import org.apache.iotdb.lsm.strategy.AccessStrategy;
+import org.apache.iotdb.lsm.strategy.IAccessStrategy;
 import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy;
 
 /** represents the context of a request */
 public class RequestContext {
 
   // memory Structure Access Policy
-  AccessStrategy accessStrategy;
+  IAccessStrategy accessStrategy;
 
   // the tree level of the currently pending memory node
   int level;
@@ -48,11 +48,11 @@ public int getLevel() {
     return level;
   }
 
-  public AccessStrategy getAccessStrategy() {
+  public IAccessStrategy getAccessStrategy() {
     return accessStrategy;
   }
 
-  public void setAccessStrategy(AccessStrategy accessStrategy) {
+  public void setAccessStrategy(IAccessStrategy accessStrategy) {
     this.accessStrategy = accessStrategy;
   }
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
new file mode 100644
index 0000000000000..6d26e77f42598
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.engine;
+
+import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.request.DeletionRequest;
+import org.apache.iotdb.lsm.request.InsertionRequest;
+import org.apache.iotdb.lsm.request.QueryRequest;
+
+import java.io.IOException;
+
+public interface ILSMEngine {
+
+   void insert(InsertionRequest insertionRequest) throws Exception;
+
+   void insert(
+      InsertionRequest insertionRequest, InsertRequestContext insertRequestContext)
+      throws Exception;
+
+   void query(QueryRequest queryRequest) throws Exception;
+
+   void query(QueryRequest queryRequest, QueryRequestContext queryRequestContext)
+      throws Exception;
+
+   void delete(DeletionRequest deletionRequest) throws Exception;
+
+   void delete(
+      DeletionRequest deletionRequest, DeleteRequestContext deleteRequestContext)
+      throws Exception;
+
+  void recover() throws Exception;
+
+  void clear() throws IOException;
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
new file mode 100644
index 0000000000000..7346753dc1194
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.engine;
+
+import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.manager.DeletionManager;
+import org.apache.iotdb.lsm.manager.InsertionManager;
+import org.apache.iotdb.lsm.manager.QueryManager;
+import org.apache.iotdb.lsm.manager.RecoverManager;
+import org.apache.iotdb.lsm.manager.WALManager;
+import org.apache.iotdb.lsm.recover.IRecoverable;
+import org.apache.iotdb.lsm.request.DeletionRequest;
+import org.apache.iotdb.lsm.request.InsertionRequest;
+import org.apache.iotdb.lsm.request.QueryRequest;
+import org.apache.iotdb.lsm.request.Request;
+
+import java.io.IOException;
+
+public class LSMEngine implements ILSMEngine, IRecoverable {
+
+  private InsertionManager insertionManager;
+
+  private DeletionManager deletionManager;
+
+  private QueryManager queryManager;
+
+  private WALManager walManager;
+
+  private RecoverManager> recoverManager;
+
+  private T rootMemNode;
+
+  public LSMEngine() {}
+
+  public LSMEngine(WALManager walManager) throws Exception {
+    this.walManager = walManager;
+    insertionManager = new InsertionManager<>(walManager);
+    deletionManager = new DeletionManager<>(walManager);
+    queryManager = new QueryManager<>();
+    recoverManager = new RecoverManager<>(walManager);
+    recoverManager.recover(this);
+  }
+
+  public T getRootMemNode() {
+    return rootMemNode;
+  }
+
+  public void setRootMemNode(T rootMemNode) {
+    this.rootMemNode = rootMemNode;
+  }
+
+  @Override
+  public  void insert(InsertionRequest insertionRequest) throws Exception {
+    insertionManager.process(rootMemNode, insertionRequest, new InsertRequestContext());
+  }
+
+  @Override
+  public  void insert(
+      InsertionRequest insertionRequest, InsertRequestContext insertRequestContext)
+      throws Exception {
+    insertionManager.process(rootMemNode, insertionRequest, insertRequestContext);
+  }
+
+  @Override
+  public  void query(QueryRequest queryRequest) throws Exception {
+    queryManager.process(rootMemNode, queryRequest, new QueryRequestContext());
+  }
+
+  @Override
+  public  void query(QueryRequest queryRequest, QueryRequestContext queryRequestContext)
+      throws Exception {
+    queryManager.process(rootMemNode, queryRequest, queryRequestContext);
+  }
+
+  @Override
+  public  void delete(DeletionRequest deletionRequest) throws Exception {
+    deletionManager.process(rootMemNode, deletionRequest, new DeleteRequestContext());
+  }
+
+  @Override
+  public  void delete(
+      DeletionRequest deletionRequest, DeleteRequestContext deleteRequestContext)
+      throws Exception {
+    deletionManager.process(rootMemNode, deletionRequest, deleteRequestContext);
+  }
+
+  @Override
+  public void recover() throws Exception {
+    recoverManager.recover(this);
+  }
+
+  @Override
+  public void clear() throws IOException {
+    walManager.close();
+  }
+
+  public InsertionManager getInsertionManager() {
+    return insertionManager;
+  }
+
+  public  void setInsertionManager(
+      InsertionManager insertionManager) {
+    this.insertionManager = (InsertionManager) insertionManager;
+  }
+
+  public DeletionManager getDeletionManager() {
+    return deletionManager;
+  }
+
+  public  void setDeletionManager(
+      DeletionManager deletionManager) {
+    this.deletionManager = (DeletionManager) deletionManager;
+  }
+
+  public QueryManager getQueryManager() {
+    return queryManager;
+  }
+
+  public  void setQueryManager(QueryManager queryManager) {
+    this.queryManager = (QueryManager) queryManager;
+  }
+
+  public WALManager getWalManager() {
+    return walManager;
+  }
+
+  public void setWalManager(WALManager walManager) {
+    this.walManager = walManager;
+  }
+
+  public RecoverManager> getRecoverManager() {
+    return recoverManager;
+  }
+
+  public void setRecoverManager(RecoverManager> recoverManager) {
+    this.recoverManager = recoverManager;
+  }
+
+  @Override
+  public  void recover(Request request) throws Exception {
+    switch (request.getRequestType()) {
+      case INSERT:
+        insert((InsertionRequest) request);
+        break;
+      case DELETE:
+        delete((DeletionRequest) request);
+        break;
+      default:
+        break;
+    }
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
index 7f06ef5be133c..0173f3ab3866e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
@@ -24,10 +24,10 @@
 
 /** the processing method corresponding to each layer of memory nodes */
 public abstract class BasicLevelProcess
-    implements LevelProcess {
+    implements ILevelProcess {
 
   // the next level process
-  LevelProcess next;
+  ILevelProcess next;
 
   /**
    * process the current layer memory node
@@ -53,7 +53,7 @@ public abstract class BasicLevelProcess
    * @return the next level process
    */
   @Override
-  public  LevelProcess nextLevel(LevelProcess next) {
+  public  ILevelProcess nextLevel(ILevelProcess next) {
     this.next = next;
     return next;
   }
@@ -73,7 +73,7 @@ public boolean hasNext() {
     return next != null;
   }
 
-  public LevelProcess getNext() {
+  public ILevelProcess getNext() {
     return next;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcess.java
similarity index 90%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcess.java
index 1eb0379906c13..38368d32c3102 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcess.java
@@ -21,7 +21,7 @@
 import org.apache.iotdb.lsm.context.RequestContext;
 
 /** the processing method corresponding to each layer of memory nodes */
-public interface LevelProcess {
+public interface ILevelProcess {
 
   /**
    * add the LevelProcess of the next layer of memory nodes
@@ -29,7 +29,7 @@ public interface LevelProcess {
    * @param next LevelProcess of the next layer
    * @return LevelProcess of the next layer
    */
-   LevelProcess nextLevel(LevelProcess next);
+   ILevelProcess nextLevel(ILevelProcess next);
 
   /**
    * use this method to process memory nodes at each layer according to the access strategy
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
index 9f34eac6413a8..8c2dae4713c44 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
@@ -19,13 +19,14 @@
 package org.apache.iotdb.lsm.levelProcess;
 
 import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.request.Request;
 
-public class LevelProcessChain {
+public class LevelProcessChain {
 
   // the level process of the first layer of memory nodes
-  LevelProcess headLevelProcess;
+  ILevelProcess headLevelProcess;
 
-  public  LevelProcess nextLevel(LevelProcess next) {
+  public  ILevelProcess nextLevel(ILevelProcess next) {
     this.headLevelProcess = next;
     return next;
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
index 175bf334d4e50..47ec4bb917d1b 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
@@ -20,9 +20,11 @@
 
 import org.apache.iotdb.lsm.context.RequestContext;
 import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.request.Request;
 
-/** */
-public class BasicLSMManager implements LSMManager {
+/** basic lsm manager */
+public abstract class BasicLSMManager
+    implements ILSMManager {
 
   // the level process of the first layer of memory nodes
   LevelProcessChain levelProcessChain;
@@ -33,26 +35,6 @@ public BasicLSMManager(LevelProcessChain levelProcessChain) {
     this.levelProcessChain = levelProcessChain;
   }
 
-  /**
-   * preprocessing of the root memory node
-   *
-   * @param root root memory node
-   * @param context request context
-   * @throws Exception
-   */
-  @Override
-  public void preProcess(T root, R request, C context) throws Exception {}
-
-  /**
-   * postprocessing of the root memory node
-   *
-   * @param root root memory node
-   * @param context request context
-   * @throws Exception
-   */
-  @Override
-  public void postProcess(T root, R request, C context) throws Exception {}
-
   /**
    * processing of the root memory node
    *
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
new file mode 100644
index 0000000000000..32974a7a5f0ee
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.manager;
+
+import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.request.DeletionRequest;
+
+/** manage deletion to MemTable */
+public class DeletionManager
+    extends BasicLSMManager {
+
+  // use wal manager object to write wal file on deletion
+  private WALManager walManager;
+
+  public DeletionManager(WALManager walManager) {
+    this.walManager = walManager;
+  }
+
+  /**
+   * write wal file on deletion
+   *
+   * @param root root memory node
+   * @param context request context
+   * @throws Exception
+   */
+  @Override
+  public void preProcess(T root, R deletionRequest, DeleteRequestContext context) throws Exception {
+    walManager.write(deletionRequest);
+  }
+
+  @Override
+  public void postProcess(T root, R request, DeleteRequestContext context) throws Exception {}
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java
similarity index 96%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java
index 3783f7f14c513..1d6afa9879865 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/LSMManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java
@@ -21,7 +21,7 @@
 import org.apache.iotdb.lsm.context.RequestContext;
 
 // used to implement lsm manager
-public interface LSMManager {
+public interface ILSMManager {
 
   /**
    * preprocessing of the root memory node
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
new file mode 100644
index 0000000000000..69547c48a3c5d
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.manager;
+
+import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.request.InsertionRequest;
+
+import java.io.IOException;
+
+/** manage insertion to MemTable */
+public class InsertionManager
+    extends BasicLSMManager {
+
+  // use wal manager object to write wal file on insertion
+  private WALManager walManager;
+
+  public InsertionManager(WALManager walManager) {
+    this.walManager = walManager;
+  }
+
+  /**
+   * write wal file on insertion
+   *
+   * @param root root memory node
+   * @param context insert request context
+   * @throws Exception
+   */
+  @Override
+  public void preProcess(T root, R insertionRequest, InsertRequestContext context)
+      throws IOException {
+    walManager.write(insertionRequest);
+  }
+
+  @Override
+  public void postProcess(T root, R request, InsertRequestContext context) throws Exception {}
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
new file mode 100644
index 0000000000000..3f828265d227a
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.manager;
+
+import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.request.QueryRequest;
+
+/** manage query to MemTable */
+public class QueryManager
+    extends BasicLSMManager {
+  @Override
+  public void preProcess(T root, R request, QueryRequestContext context) throws Exception {}
+
+  @Override
+  public void postProcess(T root, R request, QueryRequestContext context) throws Exception {}
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
new file mode 100644
index 0000000000000..c6df6cebd56a6
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.manager;
+
+import org.apache.iotdb.lsm.recover.IRecoverable;
+import org.apache.iotdb.lsm.request.Request;
+
+/** for memory structure recovery */
+public class RecoverManager {
+
+  private WALManager walManager;
+
+  public RecoverManager(WALManager walManager) {
+    this.walManager = walManager;
+    walManager.setRecover(true);
+  }
+
+  /**
+   * recover
+   *
+   * @param t
+   */
+  public void recover(T t) throws Exception {
+    while (true) {
+      Request request = walManager.read();
+      if (request == null) {
+        walManager.setRecover(false);
+        return;
+      }
+      t.recover(request);
+    }
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
new file mode 100644
index 0000000000000..c5c8ba42651e6
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.manager;
+
+import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.wal.WALReader;
+import org.apache.iotdb.lsm.wal.WALRecord;
+import org.apache.iotdb.lsm.wal.WALWriter;
+
+import java.io.File;
+import java.io.IOException;
+
+/** Manage wal entry writes and reads */
+public abstract class WALManager {
+
+  private final String schemaDirPath;
+
+  private File walFile;
+
+  // directly use the wal writer that comes with the lsm framework
+  private WALWriter walWriter;
+
+  // directly use the wal reader that comes with the lsm framework
+  private WALReader walReader;
+
+  private boolean recover;
+
+  public WALManager(String schemaDirPath) {
+    this.schemaDirPath = schemaDirPath;
+  }
+
+  public WALManager(
+      String schemaDirPath,
+      String walFileName,
+      int walBufferSize,
+      WALRecord walRecord,
+      boolean forceEachWrite)
+      throws IOException {
+    this.schemaDirPath = schemaDirPath;
+    initFile(schemaDirPath, walFileName);
+    walWriter = new WALWriter(walFile, walBufferSize, forceEachWrite);
+    walReader = new WALReader(walFile, walRecord);
+    recover = false;
+  }
+
+  private void initFile(String schemaDirPath, String walFileName) throws IOException {
+    File schemaDir = new File(schemaDirPath);
+    schemaDir.mkdirs();
+    walFile = new File(this.schemaDirPath, walFileName);
+    if (!walFile.exists()) {
+      walFile.createNewFile();
+    }
+  }
+
+  /**
+   * handle wal log writes for each request
+   *
+   * @param request request context
+   * @throws IOException
+   */
+  public abstract void write(Request request) throws IOException;
+
+  /**
+   * for recover
+   *
+   * @return request
+   */
+  public abstract Request read();
+
+  public void close() throws IOException {
+    walWriter.close();
+    walReader.close();
+  }
+
+  public String getSchemaDirPath() {
+    return schemaDirPath;
+  }
+
+  public File getWalFile() {
+    return walFile;
+  }
+
+  public void setWalFile(File walFile) {
+    this.walFile = walFile;
+  }
+
+  public WALWriter getWalWriter() {
+    return walWriter;
+  }
+
+  public void setWalWriter(WALWriter walWriter) {
+    this.walWriter = walWriter;
+  }
+
+  public WALReader getWalReader() {
+    return walReader;
+  }
+
+  public void setWalReader(WALReader walReader) {
+    this.walReader = walReader;
+  }
+
+  public boolean isRecover() {
+    return recover;
+  }
+
+  public void setRecover(boolean recover) {
+    this.recover = recover;
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
new file mode 100644
index 0000000000000..b0ea7441f43e2
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.recover;
+
+import org.apache.iotdb.lsm.request.Request;
+
+public interface IRecoverable {
+
+   void recover(Request Request) throws Exception;
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java
index 36f2ed69b112c..48dd02286b31b 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.request;
 
-public abstract class DeletionRequest extends Request {
+public abstract class DeletionRequest extends Request {
 
   public DeletionRequest() {
     requestType = RequestType.DELETE;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java
index 80e32626ac339..bfc4c2f11d7a4 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.request;
 
-public abstract class InsertionRequest extends Request {
+public abstract class InsertionRequest extends Request {
 
   public InsertionRequest() {
     requestType = RequestType.INSERT;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java
index 9ad69df9d067d..3cdcb7ef145b6 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.request;
 
-public abstract class QueryRequest extends Request {
+public abstract class QueryRequest extends Request {
 
   public QueryRequest() {
     requestType = RequestType.QUERY;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java
index dd6fcef56ab1a..3ce5c516ba020 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java
@@ -22,7 +22,7 @@
 
 import java.util.List;
 
-public abstract class Request {
+public abstract class Request {
 
   RequestType requestType;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
index 28075beb0d6a7..1bb929e53d2eb 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
@@ -27,7 +27,7 @@
 import java.util.Queue;
 
 /** breadth-first access strategy implementation class */
-public class BFSAccessStrategy implements AccessStrategy {
+public class BFSAccessStrategy implements IAccessStrategy {
 
   // same level memory nodes, used to implement BFSAccessStrategy
   Queue sameLevelMemNodes;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java
similarity index 97%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java
index bf45715f0482f..e78a7b2ca00ab 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java
@@ -22,7 +22,7 @@
 import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess;
 
 /** access strategy for memory nodes */
-public interface AccessStrategy {
+public interface IAccessStrategy {
 
   /**
    * implementation of access strategy
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
index 86986f41c5c6b..b04889ca02be0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
@@ -24,7 +24,7 @@
 import java.util.List;
 
 /** post-order traversal access strategy implementation class */
-public class PostOrderAccessStrategy implements AccessStrategy {
+public class PostOrderAccessStrategy implements IAccessStrategy {
 
   /**
    * post-order traversal access strategy
@@ -37,7 +37,7 @@ public class PostOrderAccessStrategy implements AccessStrategy {
   public  void execute(
       BasicLevelProcess levelProcess, I memNode, R request, C context) {
     int currentLevel = context.getLevel();
-    AccessStrategy accessStrategy = context.getAccessStrategy();
+    IAccessStrategy accessStrategy = context.getAccessStrategy();
     // get all memory nodes to be processed in the next layer
     List children = levelProcess.getChildren(memNode, request, context);
     if (levelProcess.hasNext()) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
index cb423d5b44a7d..c6b5ba76dbda1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
@@ -24,7 +24,7 @@
 import java.util.List;
 
 /** pre-order traversal access strategy implementation class */
-public class PreOrderAccessStrategy implements AccessStrategy {
+public class PreOrderAccessStrategy implements IAccessStrategy {
 
   /**
    * pre-order traversal access strategy
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
index b6b0c14a28a89..f89861faa78a1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
@@ -24,7 +24,7 @@
 import java.util.List;
 
 /** reverse breadth first traversal access strategy implementation class */
-public class RBFSAccessStrategy implements AccessStrategy {
+public class RBFSAccessStrategy implements IAccessStrategy {
 
   /**
    * reverse breadth first traversal access strategy
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
index 32175edc9c9bf..1eef0577f454d 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
@@ -21,9 +21,10 @@
 import java.io.DataInputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.List;
 
 /** represents a wal record, which can be extended to implement more complex wal records */
-public interface WALRecord extends Cloneable {
+public interface WALRecord extends Cloneable {
 
   /**
    * serialize the wal record
@@ -42,4 +43,8 @@ public interface WALRecord extends Cloneable {
 
   // generate wal record using prototyping pattern
   WALRecord clone();
+
+  List getKeys();
+
+  V getValue();
 }
diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndexTest.java
new file mode 100644
index 0000000000000..e92bfdd42005d
--- /dev/null
+++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndexTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
+
+import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
+import org.apache.iotdb.tsfile.utils.Pair;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class InvertedIndexTest {
+  private String[][] record =
+      new String[][] {
+        {"tag1=q", "tag2=a", "1"},
+        {"tag1=q", "tag2=s", "2"},
+        {"tag1=q", "tag2=a", "tag3=z", "3"},
+        {"tag1=q", "tag3=v", "4"},
+        {"tag1=q", "tag2=s", "5"},
+        {"tag1=w", "tag2=d", "6"},
+        {"tag1=q", "tag2=d", "tag3=e", "7"},
+        {"tag1=t", "tag2=g", "8"},
+        {"tag1=r", "tag2=d", "9"},
+        {"tag1=t", "tag2=f", "10"},
+        {"tag1=t", "tag2=h", "11"},
+        {"tag1=q", "tag2=a", "tag3=l", "12"},
+        {"tag1=y", "tag2=j", "13"},
+        {"tag1=u", "tag2=k", "14"},
+        {"tag1=q", "tag2=a", "tag3=x", "15"},
+        {"tag1=q", "tag2=a", "tag4=z", "16"},
+        {"tag1=y", "tag2=a", "tag4=z", "17"},
+        {"tag1=q", "tag2=b", "tag3=x", "18"},
+      };
+
+  private int numOfDeviceIdsInMemTable;
+
+  private InvertedIndex tagInvertedIndex;
+
+  private String storageGroupDirPath;
+
+  private String schemaRegionDirPath;
+
+  private String storageGroupFullPath = "root/testTagIndex";
+
+  private String schemaDir;
+
+  @Before
+  public void setUp() throws Exception {
+    numOfDeviceIdsInMemTable =
+        TagSchemaDescriptor.getInstance().getTagSchemaConfig().getNumOfDeviceIdsInMemTable();
+    TagSchemaDescriptor.getInstance().getTagSchemaConfig().setNumOfDeviceIdsInMemTable(3);
+    schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
+    storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath;
+    schemaRegionDirPath = storageGroupDirPath + File.separator + 0;
+    tagInvertedIndex = new InvertedIndex(schemaRegionDirPath);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    TagSchemaDescriptor.getInstance()
+        .getTagSchemaConfig()
+        .setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable);
+    tagInvertedIndex.clear();
+    tagInvertedIndex = null;
+    FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir));
+  }
+
+  public void addTags() {
+    List, Integer>> records = generateTags();
+    for (Pair, Integer> pair : records) {
+      tagInvertedIndex.addTags(pair.left, pair.right);
+    }
+  }
+
+  public void removeTags() {
+    Pair, Integer> tags = generateTag(record[0]);
+    tagInvertedIndex.removeTags(tags.left, tags.right);
+    tags = generateTag(record[1]);
+    tagInvertedIndex.removeTags(tags.left, tags.right);
+    tags = generateTag(record[3]);
+    tagInvertedIndex.removeTags(tags.left, tags.right);
+    tags = generateTag(record[11]);
+    tagInvertedIndex.removeTags(tags.left, tags.right);
+  }
+
+  @Test
+  public void getMatchedIDs() {
+    addTags();
+    Map tags1 = new HashMap<>();
+    tags1.put("tag1", "q");
+
+    Map tags2 = new HashMap<>();
+    tags2.put("tag1", "q");
+    tags2.put("tag2", "a");
+
+    List ids = tagInvertedIndex.getMatchedIDs(tags1);
+    List verify = Arrays.asList(1, 2, 3, 4, 5, 7, 12, 15, 16, 18);
+    assertEquals(verify, ids);
+
+    ids = tagInvertedIndex.getMatchedIDs(tags2);
+    verify = Arrays.asList(1, 3, 12, 15, 16);
+    assertEquals(verify, ids);
+
+    removeTags();
+
+    ids = tagInvertedIndex.getMatchedIDs(tags1);
+    verify = Arrays.asList(3, 5, 7, 15, 16, 18);
+    assertEquals(verify, ids);
+
+    ids = tagInvertedIndex.getMatchedIDs(tags2);
+    verify = Arrays.asList(3, 15, 16);
+    assertEquals(verify, ids);
+  }
+
+  @Test
+  public void testRecover() throws IOException {
+    Map tags1 = new HashMap<>();
+    tags1.put("tag1", "q");
+
+    Map tags2 = new HashMap<>();
+    tags2.put("tag1", "q");
+    tags2.put("tag2", "a");
+    addTags();
+    removeTags();
+
+    tagInvertedIndex.clear();
+    tagInvertedIndex = new InvertedIndex(schemaRegionDirPath);
+
+    List ids = tagInvertedIndex.getMatchedIDs(tags1);
+    List verify = Arrays.asList(3, 5, 7, 15, 16, 18);
+    assertEquals(verify, ids);
+
+    ids = tagInvertedIndex.getMatchedIDs(tags2);
+    verify = Arrays.asList(3, 15, 16);
+    assertEquals(verify, ids);
+  }
+
+  private List, Integer>> generateTags() {
+    List, Integer>> pairs = new ArrayList<>();
+    for (String[] strings : record) {
+      pairs.add(generateTag(strings));
+    }
+    return pairs;
+  }
+
+  private Pair, Integer> generateTag(String[] strings) {
+    Map tags = new HashMap<>();
+    int i = 0;
+    for (; i < strings.length - 1; i++) {
+      String[] str = strings[i].split("=");
+      tags.put(str[0], str[1]);
+    }
+    Pair, Integer> pair = new Pair<>(tags, Integer.valueOf(strings[i]));
+    return pair;
+  }
+}

From d094cb5c890df15244cc0cff34f68a016d363758 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sun, 23 Oct 2022 14:33:51 +0800
Subject: [PATCH 38/53] LSMEngine and MemTableGroup

---
 .../tagSchemaRegion/TagSchemaRegion.java      |   6 +-
 .../tagIndex/ITagInvertedIndex.java           |  17 --
 .../tagIndex/InvertedIndex.java               | 195 --------------
 .../tagIndex/TagInvertedIndex.java            | 244 ++++++------------
 .../tagIndex/deletion/DeletionManager.java    |  68 -----
 .../tagIndex/insertion/InsertionManager.java  |  70 -----
 .../tagIndex/query/QueryManager.java          |  52 ----
 .../tagIndex/recover/RecoverManager.java      |  67 -----
 .../tagIndex/InvertedIndexTest.java           | 184 -------------
 ...Test.java => TagTagInvertedIndexTest.java} |   2 +-
 10 files changed, 82 insertions(+), 823 deletions(-)
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndex.java
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java
 delete mode 100644 schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndexTest.java
 rename schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/{TagInvertedIndexTest.java => TagTagInvertedIndexTest.java} (99%)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
index 0a0ef22541290..ac0611fbca2cc 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
@@ -55,7 +55,7 @@
 import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion;
 import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.idtable.IDTableWithDeviceIDListImpl;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.InvertedIndex;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.MeasurementPathUtils;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.PathTagConverterUtils;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.ShowTimeSeriesResultUtils;
@@ -110,7 +110,7 @@ public class TagSchemaRegion implements ISchemaRegion {
   private final String schemaRegionDirPath;
 
   // tag inverted index
-  private final InvertedIndex tagInvertedIndex;
+  private final TagInvertedIndex tagInvertedIndex;
 
   // manager timeSeries, and use a deviceID list manager device id -> INT32 id
   private final IDTableWithDeviceIDListImpl idTableWithDeviceIDList;
@@ -131,7 +131,7 @@ public TagSchemaRegion(
     this.seriesNumerLimiter = seriesNumerLimiter;
     File schemaRegionDir = new File(schemaRegionDirPath);
     idTableWithDeviceIDList = new IDTableWithDeviceIDListImpl(schemaRegionDir);
-    tagInvertedIndex = new InvertedIndex(schemaRegionDirPath);
+    tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath);
     init();
   }
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java
index 2b52195e2cef0..aaec01847cd6d 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java
@@ -18,22 +18,12 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
 
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
-
 import java.util.List;
 import java.util.Map;
 
 /** tag inverted index interface */
 public interface ITagInvertedIndex {
 
-  /**
-   * insert tags and id using insert request context
-   *
-   * @param insertionRequest insert request context
-   */
-  void addTags(InsertionRequest insertionRequest);
-
   /**
    * insert tags and device id
    *
@@ -42,13 +32,6 @@ public interface ITagInvertedIndex {
    */
   void addTags(Map tags, int id);
 
-  /**
-   * delete tags and id using delete request context
-   *
-   * @param deletionRequest delete request context
-   */
-  void removeTags(DeletionRequest deletionRequest);
-
   /**
    * delete tags and id using delete request context
    *
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndex.java
deleted file mode 100644
index f4a8f628de27d..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndex.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
-
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkGroupDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableGroupDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkGroupInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableGroupInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkGroupQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableGroupQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.engine.LSMEngine;
-import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
-import org.apache.iotdb.lsm.manager.DeletionManager;
-import org.apache.iotdb.lsm.manager.InsertionManager;
-import org.apache.iotdb.lsm.manager.QueryManager;
-import org.apache.iotdb.lsm.manager.RecoverManager;
-
-import org.roaringbitmap.RoaringBitmap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-public class InvertedIndex implements ITagInvertedIndex {
-
-  private static final String WAL_FILE_NAME = "tag_inverted_index.log";
-
-  private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class);
-
-  private static final TagSchemaConfig tagSchemaConfig =
-      TagSchemaDescriptor.getInstance().getTagSchemaConfig();
-
-  LSMEngine lsmEngine;
-
-  public InvertedIndex(String schemaDirPath) {
-    LevelProcessChain
-        insertionLevelProcessChain = new LevelProcessChain<>();
-    LevelProcessChain
-        deletionLevelProcessChain = new LevelProcessChain<>();
-    LevelProcessChain queryLevelProcessChain =
-        new LevelProcessChain<>();
-    insertionLevelProcessChain
-        .nextLevel(new MemTableGroupInsertion())
-        .nextLevel(new MemTableInsertion())
-        .nextLevel(new MemChunkGroupInsertion())
-        .nextLevel(new MemChunkInsertion());
-    deletionLevelProcessChain
-        .nextLevel(new MemTableGroupDeletion())
-        .nextLevel(new MemTableDeletion())
-        .nextLevel(new MemChunkGroupDeletion())
-        .nextLevel(new MemChunkDeletion());
-    queryLevelProcessChain
-        .nextLevel(new MemTableGroupQuery())
-        .nextLevel(new MemTableQuery())
-        .nextLevel(new MemChunkGroupQuery())
-        .nextLevel(new MemChunkQuery());
-    try {
-      WALManager walManager =
-          new WALManager(
-              schemaDirPath,
-              WAL_FILE_NAME,
-              tagSchemaConfig.getWalBufferSize(),
-              new WALEntry(),
-              false);
-      InsertionManager insertionManager =
-          new InsertionManager<>(walManager);
-      DeletionManager deletionManager =
-          new DeletionManager<>(walManager);
-      QueryManager queryManager = new QueryManager<>();
-      RecoverManager> recoverManager = new RecoverManager<>(walManager);
-      insertionManager.setLevelProcessChain(insertionLevelProcessChain);
-      deletionManager.setLevelProcessChain(deletionLevelProcessChain);
-      queryManager.setLevelProcessChain(queryLevelProcessChain);
-      lsmEngine = new LSMEngine<>();
-      lsmEngine.setDeletionManager(deletionManager);
-      lsmEngine.setInsertionManager(insertionManager);
-      lsmEngine.setQueryManager(queryManager);
-      lsmEngine.setWalManager(walManager);
-      lsmEngine.setRecoverManager(recoverManager);
-      lsmEngine.setRootMemNode(new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable()));
-      lsmEngine.recover();
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-  }
-
-  @Override
-  public void addTags(InsertionRequest insertionRequest) {}
-
-  @Override
-  public synchronized void addTags(Map tags, int id) {
-    try {
-      for (Map.Entry tag : tags.entrySet()) {
-        InsertionRequest insertionRequest =
-            new InsertionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
-        lsmEngine.insert(insertionRequest);
-      }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-  }
-
-  @Override
-  public void removeTags(DeletionRequest deletionRequest) {}
-
-  @Override
-  public synchronized void removeTags(Map tags, int id) {
-    try {
-      for (Map.Entry tag : tags.entrySet()) {
-        DeletionRequest deletionRequest =
-            new DeletionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
-        lsmEngine.delete(deletionRequest);
-      }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-  }
-
-  @Override
-  public synchronized List getMatchedIDs(Map tags) {
-    RoaringBitmap roaringBitmap = new RoaringBitmap();
-    int i = 0;
-    try {
-      for (Map.Entry tag : tags.entrySet()) {
-        RoaringBitmap rb = getMatchedIDs(tag.getKey(), tag.getValue());
-        if (rb == null) continue;
-        else {
-          if (i == 0) roaringBitmap = rb;
-          else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb);
-          i++;
-        }
-      }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-    return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList());
-  }
-
-  private List generateKeys(String tagKey, String tagValue) {
-    List keys = new ArrayList<>();
-    keys.add(tagKey);
-    keys.add(tagValue);
-    return keys;
-  }
-
-  private RoaringBitmap getMatchedIDs(String tagKey, String tagValue) throws Exception {
-    QueryRequest queryRequest = new QueryRequest(generateKeys(tagKey, tagValue));
-    lsmEngine.query(queryRequest);
-    return queryRequest.getResult();
-  }
-
-  @TestOnly
-  public void clear() throws IOException {
-    lsmEngine.clear();
-  }
-}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index 2eada5e7042f2..e32bf5870936d 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -24,15 +24,30 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover.RecoverManager;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkGroupDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableGroupDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkGroupInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableGroupInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkGroupQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableGroupQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.engine.LSMEngine;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.manager.DeletionManager;
+import org.apache.iotdb.lsm.manager.InsertionManager;
+import org.apache.iotdb.lsm.manager.QueryManager;
+import org.apache.iotdb.lsm.manager.RecoverManager;
 
 import org.roaringbitmap.RoaringBitmap;
 import org.slf4j.Logger;
@@ -41,169 +56,106 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-/** tag reverse index implementation class */
 public class TagInvertedIndex implements ITagInvertedIndex {
 
+  private static final String WAL_FILE_NAME = "tag_inverted_index.log";
+
   private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class);
 
   private static final TagSchemaConfig tagSchemaConfig =
       TagSchemaDescriptor.getInstance().getTagSchemaConfig();
 
-  private InsertionManager insertionManager;
-
-  private DeletionManager deletionManager;
-
-  private QueryManager queryManager;
-
-  private WALManager walManager;
-
-  private RecoverManager recoverManager;
-
-  // the maximum number of device ids managed by a working memTable
-  private int numOfDeviceIdsInMemTable;
-
-  // (maxDeviceID / numOfDeviceIdsInMemTable) -> MemTable
-  private Map immutableMemTables;
-
-  private MemTable workingMemTable;
-
-  // the largest device id saved by the current MemTable
-  private int maxDeviceID;
+  LSMEngine lsmEngine;
 
   public TagInvertedIndex(String schemaDirPath) {
+    LevelProcessChain
+        insertionLevelProcessChain = new LevelProcessChain<>();
+    LevelProcessChain
+        deletionLevelProcessChain = new LevelProcessChain<>();
+    LevelProcessChain queryLevelProcessChain =
+        new LevelProcessChain<>();
+    insertionLevelProcessChain
+        .nextLevel(new MemTableGroupInsertion())
+        .nextLevel(new MemTableInsertion())
+        .nextLevel(new MemChunkGroupInsertion())
+        .nextLevel(new MemChunkInsertion());
+    deletionLevelProcessChain
+        .nextLevel(new MemTableGroupDeletion())
+        .nextLevel(new MemTableDeletion())
+        .nextLevel(new MemChunkGroupDeletion())
+        .nextLevel(new MemChunkDeletion());
+    queryLevelProcessChain
+        .nextLevel(new MemTableGroupQuery())
+        .nextLevel(new MemTableQuery())
+        .nextLevel(new MemChunkGroupQuery())
+        .nextLevel(new MemChunkQuery());
     try {
-      walManager = new WALManager(schemaDirPath);
-      insertionManager = new InsertionManager(walManager);
-      deletionManager = new DeletionManager(walManager);
-      recoverManager = new RecoverManager(walManager);
-      queryManager = new QueryManager();
-      workingMemTable = new MemTable(MemTable.WORKING);
-      immutableMemTables = new HashMap<>();
-      numOfDeviceIdsInMemTable = tagSchemaConfig.getNumOfDeviceIdsInMemTable();
-      maxDeviceID = 0;
-      recover();
-    } catch (Exception e) {
-      logger.error("create TagInvertedIndex fail", e);
-    }
-  }
-
-  public synchronized void recover() {
-    recoverManager.recover(this);
-  }
-
-  /**
-   * insert tags and id using insert request context
-   *
-   * @param insertionRequest insert request context
-   */
-  @Override
-  public synchronized void addTags(InsertionRequest insertionRequest) {
-    int id = insertionRequest.getValue();
-    // if the device id can not be saved to the current working MemTable
-    if (!inWorkingMemTable(id)) {
-      workingMemTable.setStatus(MemTable.IMMUTABLE);
-      immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable);
-      workingMemTable = new MemTable(MemTable.WORKING);
-    }
-    MemTable memTable = workingMemTable;
-    maxDeviceID = id;
-    try {
-      insertionManager.process(memTable, insertionRequest, new InsertRequestContext());
+      WALManager walManager =
+          new WALManager(
+              schemaDirPath,
+              WAL_FILE_NAME,
+              tagSchemaConfig.getWalBufferSize(),
+              new WALEntry(),
+              false);
+      InsertionManager insertionManager =
+          new InsertionManager<>(walManager);
+      DeletionManager deletionManager =
+          new DeletionManager<>(walManager);
+      QueryManager queryManager = new QueryManager<>();
+      RecoverManager> recoverManager = new RecoverManager<>(walManager);
+      insertionManager.setLevelProcessChain(insertionLevelProcessChain);
+      deletionManager.setLevelProcessChain(deletionLevelProcessChain);
+      queryManager.setLevelProcessChain(queryLevelProcessChain);
+      lsmEngine = new LSMEngine<>();
+      lsmEngine.setDeletionManager(deletionManager);
+      lsmEngine.setInsertionManager(insertionManager);
+      lsmEngine.setQueryManager(queryManager);
+      lsmEngine.setWalManager(walManager);
+      lsmEngine.setRecoverManager(recoverManager);
+      lsmEngine.setRootMemNode(new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable()));
+      lsmEngine.recover();
     } catch (Exception e) {
       logger.error(e.getMessage());
     }
   }
 
-  /**
-   * insert tags and device id
-   *
-   * @param tags tags like: 
-   * @param id INT32 device id
-   */
   @Override
   public synchronized void addTags(Map tags, int id) {
-    // if the device id can not be saved to the current working MemTable
-    if (!inWorkingMemTable(id)) {
-      workingMemTable.setStatus(MemTable.IMMUTABLE);
-      immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable);
-      workingMemTable = new MemTable(MemTable.WORKING);
-    }
-    MemTable memTable = workingMemTable;
-    maxDeviceID = id;
     try {
       for (Map.Entry tag : tags.entrySet()) {
-        addTag(memTable, tag.getKey(), tag.getValue(), id);
+        InsertionRequest insertionRequest =
+            new InsertionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
+        lsmEngine.insert(insertionRequest);
       }
     } catch (Exception e) {
       logger.error(e.getMessage());
     }
   }
 
-  /**
-   * delete tags and id using delete request context
-   *
-   * @param deletionRequest delete request context
-   */
-  @Override
-  public void removeTags(DeletionRequest deletionRequest) {
-    int id = deletionRequest.getValue();
-    MemTable memTable = null;
-    if (inWorkingMemTable(id)) {
-      memTable = workingMemTable;
-    } else {
-      memTable = immutableMemTables.get(id / numOfDeviceIdsInMemTable);
-    }
-    try {
-      deletionManager.process(memTable, deletionRequest, new DeleteRequestContext());
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-  }
-
-  /**
-   * delete tags and id using delete request context
-   *
-   * @param tags tags like: 
-   * @param id INT32 device id
-   */
   @Override
   public synchronized void removeTags(Map tags, int id) {
-    List memTables = new ArrayList<>();
-    if (inWorkingMemTable(id)) {
-      memTables.add(workingMemTable);
-    } else {
-      memTables.add(immutableMemTables.get(id / numOfDeviceIdsInMemTable));
-    }
     try {
       for (Map.Entry tag : tags.entrySet()) {
-        removeTag(memTables, tag.getKey(), tag.getValue(), id);
+        DeletionRequest deletionRequest =
+            new DeletionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
+        lsmEngine.delete(deletionRequest);
       }
     } catch (Exception e) {
       logger.error(e.getMessage());
     }
   }
 
-  /**
-   * get all matching device ids
-   *
-   * @param tags tags like: 
-   * @return device ids
-   */
   @Override
   public synchronized List getMatchedIDs(Map tags) {
-    List memTables = new ArrayList<>();
-    memTables.add(workingMemTable);
-    memTables.addAll(immutableMemTables.values());
     RoaringBitmap roaringBitmap = new RoaringBitmap();
     int i = 0;
     try {
       for (Map.Entry tag : tags.entrySet()) {
-        RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue());
+        RoaringBitmap rb = getMatchedIDs(tag.getKey(), tag.getValue());
         if (rb == null) continue;
         else {
           if (i == 0) roaringBitmap = rb;
@@ -217,30 +169,6 @@ public synchronized List getMatchedIDs(Map tags) {
     return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList());
   }
 
-  @Override
-  public String toString() {
-    return "TagInvertedIndex{"
-        + "numOfDeviceIdsInMemTable="
-        + numOfDeviceIdsInMemTable
-        + ", workingMemTable="
-        + workingMemTable
-        + ", immutableMemTables="
-        + immutableMemTables
-        + ", maxDeviceID="
-        + maxDeviceID
-        + '}';
-  }
-
-  /**
-   * determine whether the id can be saved to the current MemTable
-   *
-   * @param id INT32 device id
-   * @return return true if it can, otherwise return false
-   */
-  private boolean inWorkingMemTable(int id) {
-    return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable;
-  }
-
   private List generateKeys(String tagKey, String tagValue) {
     List keys = new ArrayList<>();
     keys.add(tagKey);
@@ -248,30 +176,14 @@ private List generateKeys(String tagKey, String tagValue) {
     return keys;
   }
 
-  private void addTag(MemTable memTable, String tagKey, String tagValue, int id) throws Exception {
-    InsertionRequest insertionRequest = new InsertionRequest(generateKeys(tagKey, tagValue), id);
-    insertionManager.process(memTable, insertionRequest, new InsertRequestContext());
-  }
-
-  private void removeTag(List memTables, String tagKey, String tagValue, int id)
-      throws Exception {
-    DeletionRequest deletionRequest = new DeletionRequest(generateKeys(tagKey, tagValue), id);
-    for (MemTable memTable : memTables) {
-      deletionManager.process(memTable, deletionRequest, new DeleteRequestContext());
-    }
-  }
-
-  private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue)
-      throws Exception {
+  private RoaringBitmap getMatchedIDs(String tagKey, String tagValue) throws Exception {
     QueryRequest queryRequest = new QueryRequest(generateKeys(tagKey, tagValue));
-    for (MemTable memTable : memTables) {
-      queryManager.process(memTable, queryRequest, new QueryRequestContext());
-    }
+    lsmEngine.query(queryRequest);
     return queryRequest.getResult();
   }
 
   @TestOnly
   public void clear() throws IOException {
-    walManager.close();
+    lsmEngine.clear();
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
deleted file mode 100644
index dad8df35d8730..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion;
-
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
-import org.apache.iotdb.lsm.manager.BasicLSMManager;
-
-/** manage deletion to MemTable */
-public class DeletionManager
-    extends BasicLSMManager {
-
-  // use wal manager object to write wal file on deletion
-  private WALManager walManager;
-
-  public DeletionManager(WALManager walManager) {
-    this.walManager = walManager;
-    initLevelProcess();
-  }
-
-  /**
-   * write wal file on deletion
-   *
-   * @param root root memory node
-   * @param context request context
-   * @throws Exception
-   */
-  @Override
-  public void preProcess(
-      MemTable root, DeletionRequest deletionRequest, DeleteRequestContext context)
-      throws Exception {
-    walManager.write(deletionRequest);
-  }
-
-  @Override
-  public void postProcess(MemTable root, DeletionRequest request, DeleteRequestContext context)
-      throws Exception {}
-
-  /** set the delete operation for each layer of memory nodes */
-  private void initLevelProcess() {
-    LevelProcessChain levelProcessChain =
-        new LevelProcessChain<>();
-    levelProcessChain
-        .nextLevel(new MemTableDeletion())
-        .nextLevel(new MemChunkGroupDeletion())
-        .nextLevel(new MemChunkDeletion());
-    setLevelProcessChain(levelProcessChain);
-  }
-}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
deleted file mode 100644
index d272e3476abbf..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion;
-
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
-import org.apache.iotdb.lsm.manager.BasicLSMManager;
-
-import java.io.IOException;
-
-/** manage insertion to MemTable */
-public class InsertionManager
-    extends BasicLSMManager {
-
-  // use wal manager object to write wal file on insertion
-  private WALManager walManager;
-
-  public InsertionManager(WALManager walManager) {
-    this.walManager = walManager;
-    initLevelProcess();
-  }
-
-  /**
-   * write wal file on insertion
-   *
-   * @param root root memory node
-   * @param context insert request context
-   * @throws Exception
-   */
-  @Override
-  public void preProcess(
-      MemTable root, InsertionRequest insertionRequest, InsertRequestContext context)
-      throws IOException {
-    walManager.write(insertionRequest);
-  }
-
-  @Override
-  public void postProcess(MemTable root, InsertionRequest request, InsertRequestContext context)
-      throws Exception {}
-
-  /** set the insert operation for each layer of memory nodes */
-  private void initLevelProcess() {
-    LevelProcessChain levelProcessChain =
-        new LevelProcessChain<>();
-    levelProcessChain
-        .nextLevel(new MemTableInsertion())
-        .nextLevel(new MemChunkGroupInsertion())
-        .nextLevel(new MemChunkInsertion());
-    setLevelProcessChain(levelProcessChain);
-  }
-}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
deleted file mode 100644
index e5d97dd15e0c3..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query;
-
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
-import org.apache.iotdb.lsm.manager.BasicLSMManager;
-
-/** manage insertion to MemTable */
-public class QueryManager extends BasicLSMManager {
-
-  public QueryManager() {
-    initLevelProcess();
-  }
-
-  /** set the query operation for each layer of memory nodes */
-  private void initLevelProcess() {
-    LevelProcessChain levelProcessChain =
-        new LevelProcessChain<>();
-    levelProcessChain
-        .nextLevel(new MemTableQuery())
-        .nextLevel(new MemChunkGroupQuery())
-        .nextLevel(new MemChunkQuery());
-    setLevelProcessChain(levelProcessChain);
-  }
-
-  @Override
-  public void preProcess(MemTable root, QueryRequest request, QueryRequestContext context)
-      throws Exception {}
-
-  @Override
-  public void postProcess(MemTable root, QueryRequest request, QueryRequestContext context)
-      throws Exception {}
-}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java
deleted file mode 100644
index cd3bda3d43752..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover;
-
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.ITagInvertedIndex;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
-import org.apache.iotdb.lsm.request.Request;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** for memory structure recovery */
-public class RecoverManager {
-  private static final Logger logger = LoggerFactory.getLogger(RecoverManager.class);
-
-  private WALManager walManager;
-
-  public RecoverManager(WALManager walManager) {
-    this.walManager = walManager;
-    walManager.setRecover(true);
-  }
-
-  /**
-   * recover tagInvertedIndex
-   *
-   * @param tagInvertedIndex tag inverted index
-   */
-  public void recover(ITagInvertedIndex tagInvertedIndex) {
-    logger.info("recover tagInvertedIndex");
-    while (true) {
-      Request request = walManager.read();
-      if (request == null) {
-        walManager.setRecover(false);
-        return;
-      }
-      switch (request.getRequestType()) {
-        case INSERT:
-          tagInvertedIndex.addTags((InsertionRequest) request);
-          break;
-        case DELETE:
-          tagInvertedIndex.removeTags((DeletionRequest) request);
-          break;
-        default:
-          walManager.setRecover(false);
-          return;
-      }
-    }
-  }
-}
diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndexTest.java
deleted file mode 100644
index e92bfdd42005d..0000000000000
--- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/InvertedIndexTest.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
-
-import org.apache.iotdb.commons.utils.FileUtils;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
-import org.apache.iotdb.tsfile.utils.Pair;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-public class InvertedIndexTest {
-  private String[][] record =
-      new String[][] {
-        {"tag1=q", "tag2=a", "1"},
-        {"tag1=q", "tag2=s", "2"},
-        {"tag1=q", "tag2=a", "tag3=z", "3"},
-        {"tag1=q", "tag3=v", "4"},
-        {"tag1=q", "tag2=s", "5"},
-        {"tag1=w", "tag2=d", "6"},
-        {"tag1=q", "tag2=d", "tag3=e", "7"},
-        {"tag1=t", "tag2=g", "8"},
-        {"tag1=r", "tag2=d", "9"},
-        {"tag1=t", "tag2=f", "10"},
-        {"tag1=t", "tag2=h", "11"},
-        {"tag1=q", "tag2=a", "tag3=l", "12"},
-        {"tag1=y", "tag2=j", "13"},
-        {"tag1=u", "tag2=k", "14"},
-        {"tag1=q", "tag2=a", "tag3=x", "15"},
-        {"tag1=q", "tag2=a", "tag4=z", "16"},
-        {"tag1=y", "tag2=a", "tag4=z", "17"},
-        {"tag1=q", "tag2=b", "tag3=x", "18"},
-      };
-
-  private int numOfDeviceIdsInMemTable;
-
-  private InvertedIndex tagInvertedIndex;
-
-  private String storageGroupDirPath;
-
-  private String schemaRegionDirPath;
-
-  private String storageGroupFullPath = "root/testTagIndex";
-
-  private String schemaDir;
-
-  @Before
-  public void setUp() throws Exception {
-    numOfDeviceIdsInMemTable =
-        TagSchemaDescriptor.getInstance().getTagSchemaConfig().getNumOfDeviceIdsInMemTable();
-    TagSchemaDescriptor.getInstance().getTagSchemaConfig().setNumOfDeviceIdsInMemTable(3);
-    schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
-    storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath;
-    schemaRegionDirPath = storageGroupDirPath + File.separator + 0;
-    tagInvertedIndex = new InvertedIndex(schemaRegionDirPath);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    TagSchemaDescriptor.getInstance()
-        .getTagSchemaConfig()
-        .setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable);
-    tagInvertedIndex.clear();
-    tagInvertedIndex = null;
-    FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir));
-  }
-
-  public void addTags() {
-    List, Integer>> records = generateTags();
-    for (Pair, Integer> pair : records) {
-      tagInvertedIndex.addTags(pair.left, pair.right);
-    }
-  }
-
-  public void removeTags() {
-    Pair, Integer> tags = generateTag(record[0]);
-    tagInvertedIndex.removeTags(tags.left, tags.right);
-    tags = generateTag(record[1]);
-    tagInvertedIndex.removeTags(tags.left, tags.right);
-    tags = generateTag(record[3]);
-    tagInvertedIndex.removeTags(tags.left, tags.right);
-    tags = generateTag(record[11]);
-    tagInvertedIndex.removeTags(tags.left, tags.right);
-  }
-
-  @Test
-  public void getMatchedIDs() {
-    addTags();
-    Map tags1 = new HashMap<>();
-    tags1.put("tag1", "q");
-
-    Map tags2 = new HashMap<>();
-    tags2.put("tag1", "q");
-    tags2.put("tag2", "a");
-
-    List ids = tagInvertedIndex.getMatchedIDs(tags1);
-    List verify = Arrays.asList(1, 2, 3, 4, 5, 7, 12, 15, 16, 18);
-    assertEquals(verify, ids);
-
-    ids = tagInvertedIndex.getMatchedIDs(tags2);
-    verify = Arrays.asList(1, 3, 12, 15, 16);
-    assertEquals(verify, ids);
-
-    removeTags();
-
-    ids = tagInvertedIndex.getMatchedIDs(tags1);
-    verify = Arrays.asList(3, 5, 7, 15, 16, 18);
-    assertEquals(verify, ids);
-
-    ids = tagInvertedIndex.getMatchedIDs(tags2);
-    verify = Arrays.asList(3, 15, 16);
-    assertEquals(verify, ids);
-  }
-
-  @Test
-  public void testRecover() throws IOException {
-    Map tags1 = new HashMap<>();
-    tags1.put("tag1", "q");
-
-    Map tags2 = new HashMap<>();
-    tags2.put("tag1", "q");
-    tags2.put("tag2", "a");
-    addTags();
-    removeTags();
-
-    tagInvertedIndex.clear();
-    tagInvertedIndex = new InvertedIndex(schemaRegionDirPath);
-
-    List ids = tagInvertedIndex.getMatchedIDs(tags1);
-    List verify = Arrays.asList(3, 5, 7, 15, 16, 18);
-    assertEquals(verify, ids);
-
-    ids = tagInvertedIndex.getMatchedIDs(tags2);
-    verify = Arrays.asList(3, 15, 16);
-    assertEquals(verify, ids);
-  }
-
-  private List, Integer>> generateTags() {
-    List, Integer>> pairs = new ArrayList<>();
-    for (String[] strings : record) {
-      pairs.add(generateTag(strings));
-    }
-    return pairs;
-  }
-
-  private Pair, Integer> generateTag(String[] strings) {
-    Map tags = new HashMap<>();
-    int i = 0;
-    for (; i < strings.length - 1; i++) {
-      String[] str = strings[i].split("=");
-      tags.put(str[0], str[1]);
-    }
-    Pair, Integer> pair = new Pair<>(tags, Integer.valueOf(strings[i]));
-    return pair;
-  }
-}
diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagTagInvertedIndexTest.java
similarity index 99%
rename from schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java
rename to schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagTagInvertedIndexTest.java
index 53151db4a0a74..b46d863b171c3 100644
--- a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java
+++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagTagInvertedIndexTest.java
@@ -37,7 +37,7 @@
 
 import static org.junit.Assert.assertEquals;
 
-public class TagInvertedIndexTest {
+public class TagTagInvertedIndexTest {
   private String[][] record =
       new String[][] {
         {"tag1=q", "tag2=a", "1"},

From 162c17bfd8ab30ef371318d17682af42514b3144 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sun, 23 Oct 2022 14:34:07 +0800
Subject: [PATCH 39/53] LSMEngine and MemTableGroup

---
 .../tagIndex/TagInvertedIndex.java            | 189 ------------------
 1 file changed, 189 deletions(-)
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
deleted file mode 100644
index e32bf5870936d..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
-
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkGroupDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableGroupDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkGroupInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableGroupInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkGroupQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableGroupQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.engine.LSMEngine;
-import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
-import org.apache.iotdb.lsm.manager.DeletionManager;
-import org.apache.iotdb.lsm.manager.InsertionManager;
-import org.apache.iotdb.lsm.manager.QueryManager;
-import org.apache.iotdb.lsm.manager.RecoverManager;
-
-import org.roaringbitmap.RoaringBitmap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-public class TagInvertedIndex implements ITagInvertedIndex {
-
-  private static final String WAL_FILE_NAME = "tag_inverted_index.log";
-
-  private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class);
-
-  private static final TagSchemaConfig tagSchemaConfig =
-      TagSchemaDescriptor.getInstance().getTagSchemaConfig();
-
-  LSMEngine lsmEngine;
-
-  public TagInvertedIndex(String schemaDirPath) {
-    LevelProcessChain
-        insertionLevelProcessChain = new LevelProcessChain<>();
-    LevelProcessChain
-        deletionLevelProcessChain = new LevelProcessChain<>();
-    LevelProcessChain queryLevelProcessChain =
-        new LevelProcessChain<>();
-    insertionLevelProcessChain
-        .nextLevel(new MemTableGroupInsertion())
-        .nextLevel(new MemTableInsertion())
-        .nextLevel(new MemChunkGroupInsertion())
-        .nextLevel(new MemChunkInsertion());
-    deletionLevelProcessChain
-        .nextLevel(new MemTableGroupDeletion())
-        .nextLevel(new MemTableDeletion())
-        .nextLevel(new MemChunkGroupDeletion())
-        .nextLevel(new MemChunkDeletion());
-    queryLevelProcessChain
-        .nextLevel(new MemTableGroupQuery())
-        .nextLevel(new MemTableQuery())
-        .nextLevel(new MemChunkGroupQuery())
-        .nextLevel(new MemChunkQuery());
-    try {
-      WALManager walManager =
-          new WALManager(
-              schemaDirPath,
-              WAL_FILE_NAME,
-              tagSchemaConfig.getWalBufferSize(),
-              new WALEntry(),
-              false);
-      InsertionManager insertionManager =
-          new InsertionManager<>(walManager);
-      DeletionManager deletionManager =
-          new DeletionManager<>(walManager);
-      QueryManager queryManager = new QueryManager<>();
-      RecoverManager> recoverManager = new RecoverManager<>(walManager);
-      insertionManager.setLevelProcessChain(insertionLevelProcessChain);
-      deletionManager.setLevelProcessChain(deletionLevelProcessChain);
-      queryManager.setLevelProcessChain(queryLevelProcessChain);
-      lsmEngine = new LSMEngine<>();
-      lsmEngine.setDeletionManager(deletionManager);
-      lsmEngine.setInsertionManager(insertionManager);
-      lsmEngine.setQueryManager(queryManager);
-      lsmEngine.setWalManager(walManager);
-      lsmEngine.setRecoverManager(recoverManager);
-      lsmEngine.setRootMemNode(new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable()));
-      lsmEngine.recover();
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-  }
-
-  @Override
-  public synchronized void addTags(Map tags, int id) {
-    try {
-      for (Map.Entry tag : tags.entrySet()) {
-        InsertionRequest insertionRequest =
-            new InsertionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
-        lsmEngine.insert(insertionRequest);
-      }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-  }
-
-  @Override
-  public synchronized void removeTags(Map tags, int id) {
-    try {
-      for (Map.Entry tag : tags.entrySet()) {
-        DeletionRequest deletionRequest =
-            new DeletionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
-        lsmEngine.delete(deletionRequest);
-      }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-  }
-
-  @Override
-  public synchronized List getMatchedIDs(Map tags) {
-    RoaringBitmap roaringBitmap = new RoaringBitmap();
-    int i = 0;
-    try {
-      for (Map.Entry tag : tags.entrySet()) {
-        RoaringBitmap rb = getMatchedIDs(tag.getKey(), tag.getValue());
-        if (rb == null) continue;
-        else {
-          if (i == 0) roaringBitmap = rb;
-          else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb);
-          i++;
-        }
-      }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-    return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList());
-  }
-
-  private List generateKeys(String tagKey, String tagValue) {
-    List keys = new ArrayList<>();
-    keys.add(tagKey);
-    keys.add(tagValue);
-    return keys;
-  }
-
-  private RoaringBitmap getMatchedIDs(String tagKey, String tagValue) throws Exception {
-    QueryRequest queryRequest = new QueryRequest(generateKeys(tagKey, tagValue));
-    lsmEngine.query(queryRequest);
-    return queryRequest.getResult();
-  }
-
-  @TestOnly
-  public void clear() throws IOException {
-    lsmEngine.clear();
-  }
-}

From 3da61b7e100c5ae66f1c00c7dba5c07b151d2412 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sun, 23 Oct 2022 14:34:31 +0800
Subject: [PATCH 40/53] LSMEngine and MemTableGroup

---
 .../tagIndex/TagInvertedIndex.java            | 189 ++++++++++++++++++
 1 file changed, 189 insertions(+)
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
new file mode 100644
index 0000000000000..e32bf5870936d
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
+
+import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkGroupDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableGroupDeletion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkGroupInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableGroupInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableInsertion;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkGroupQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableGroupQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableQuery;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
+import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.engine.LSMEngine;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.manager.DeletionManager;
+import org.apache.iotdb.lsm.manager.InsertionManager;
+import org.apache.iotdb.lsm.manager.QueryManager;
+import org.apache.iotdb.lsm.manager.RecoverManager;
+
+import org.roaringbitmap.RoaringBitmap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class TagInvertedIndex implements ITagInvertedIndex {
+
+  private static final String WAL_FILE_NAME = "tag_inverted_index.log";
+
+  private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class);
+
+  private static final TagSchemaConfig tagSchemaConfig =
+      TagSchemaDescriptor.getInstance().getTagSchemaConfig();
+
+  LSMEngine lsmEngine;
+
+  public TagInvertedIndex(String schemaDirPath) {
+    LevelProcessChain
+        insertionLevelProcessChain = new LevelProcessChain<>();
+    LevelProcessChain
+        deletionLevelProcessChain = new LevelProcessChain<>();
+    LevelProcessChain queryLevelProcessChain =
+        new LevelProcessChain<>();
+    insertionLevelProcessChain
+        .nextLevel(new MemTableGroupInsertion())
+        .nextLevel(new MemTableInsertion())
+        .nextLevel(new MemChunkGroupInsertion())
+        .nextLevel(new MemChunkInsertion());
+    deletionLevelProcessChain
+        .nextLevel(new MemTableGroupDeletion())
+        .nextLevel(new MemTableDeletion())
+        .nextLevel(new MemChunkGroupDeletion())
+        .nextLevel(new MemChunkDeletion());
+    queryLevelProcessChain
+        .nextLevel(new MemTableGroupQuery())
+        .nextLevel(new MemTableQuery())
+        .nextLevel(new MemChunkGroupQuery())
+        .nextLevel(new MemChunkQuery());
+    try {
+      WALManager walManager =
+          new WALManager(
+              schemaDirPath,
+              WAL_FILE_NAME,
+              tagSchemaConfig.getWalBufferSize(),
+              new WALEntry(),
+              false);
+      InsertionManager insertionManager =
+          new InsertionManager<>(walManager);
+      DeletionManager deletionManager =
+          new DeletionManager<>(walManager);
+      QueryManager queryManager = new QueryManager<>();
+      RecoverManager> recoverManager = new RecoverManager<>(walManager);
+      insertionManager.setLevelProcessChain(insertionLevelProcessChain);
+      deletionManager.setLevelProcessChain(deletionLevelProcessChain);
+      queryManager.setLevelProcessChain(queryLevelProcessChain);
+      lsmEngine = new LSMEngine<>();
+      lsmEngine.setDeletionManager(deletionManager);
+      lsmEngine.setInsertionManager(insertionManager);
+      lsmEngine.setQueryManager(queryManager);
+      lsmEngine.setWalManager(walManager);
+      lsmEngine.setRecoverManager(recoverManager);
+      lsmEngine.setRootMemNode(new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable()));
+      lsmEngine.recover();
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+  }
+
+  @Override
+  public synchronized void addTags(Map tags, int id) {
+    try {
+      for (Map.Entry tag : tags.entrySet()) {
+        InsertionRequest insertionRequest =
+            new InsertionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
+        lsmEngine.insert(insertionRequest);
+      }
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+  }
+
+  @Override
+  public synchronized void removeTags(Map tags, int id) {
+    try {
+      for (Map.Entry tag : tags.entrySet()) {
+        DeletionRequest deletionRequest =
+            new DeletionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
+        lsmEngine.delete(deletionRequest);
+      }
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+  }
+
+  @Override
+  public synchronized List getMatchedIDs(Map tags) {
+    RoaringBitmap roaringBitmap = new RoaringBitmap();
+    int i = 0;
+    try {
+      for (Map.Entry tag : tags.entrySet()) {
+        RoaringBitmap rb = getMatchedIDs(tag.getKey(), tag.getValue());
+        if (rb == null) continue;
+        else {
+          if (i == 0) roaringBitmap = rb;
+          else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb);
+          i++;
+        }
+      }
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+    return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList());
+  }
+
+  private List generateKeys(String tagKey, String tagValue) {
+    List keys = new ArrayList<>();
+    keys.add(tagKey);
+    keys.add(tagValue);
+    return keys;
+  }
+
+  private RoaringBitmap getMatchedIDs(String tagKey, String tagValue) throws Exception {
+    QueryRequest queryRequest = new QueryRequest(generateKeys(tagKey, tagValue));
+    lsmEngine.query(queryRequest);
+    return queryRequest.getResult();
+  }
+
+  @TestOnly
+  public void clear() throws IOException {
+    lsmEngine.clear();
+  }
+}

From 52c6912f4f7d857692fe0f010d49d4443bb412ba Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sun, 23 Oct 2022 16:59:13 +0800
Subject: [PATCH 41/53] add Property

---
 .../tagIndex/TagInvertedIndex.java            |  72 +++++-------
 .../tagIndex/memtable/MemTableGroup.java      |   2 +
 .../iotdb/lsm/engine/LSMEngineBuilder.java    | 108 ++++++++++++++++++
 .../iotdb/lsm/engine/LSMEngineDirector.java   | 103 +++++++++++++++++
 .../apache/iotdb/lsm/property/Property.java   |  54 +++++++++
 5 files changed, 295 insertions(+), 44 deletions(-)
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/Property.java

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index e32bf5870936d..4f7ef1a72d103 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -39,15 +39,9 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableQuery;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
 import org.apache.iotdb.lsm.engine.LSMEngine;
-import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
-import org.apache.iotdb.lsm.manager.DeletionManager;
-import org.apache.iotdb.lsm.manager.InsertionManager;
-import org.apache.iotdb.lsm.manager.QueryManager;
-import org.apache.iotdb.lsm.manager.RecoverManager;
+import org.apache.iotdb.lsm.engine.LSMEngineDirector;
+import org.apache.iotdb.lsm.property.Property;
 
 import org.roaringbitmap.RoaringBitmap;
 import org.slf4j.Logger;
@@ -72,27 +66,30 @@ public class TagInvertedIndex implements ITagInvertedIndex {
   LSMEngine lsmEngine;
 
   public TagInvertedIndex(String schemaDirPath) {
-    LevelProcessChain
-        insertionLevelProcessChain = new LevelProcessChain<>();
-    LevelProcessChain
-        deletionLevelProcessChain = new LevelProcessChain<>();
-    LevelProcessChain queryLevelProcessChain =
-        new LevelProcessChain<>();
-    insertionLevelProcessChain
-        .nextLevel(new MemTableGroupInsertion())
-        .nextLevel(new MemTableInsertion())
-        .nextLevel(new MemChunkGroupInsertion())
-        .nextLevel(new MemChunkInsertion());
-    deletionLevelProcessChain
-        .nextLevel(new MemTableGroupDeletion())
-        .nextLevel(new MemTableDeletion())
-        .nextLevel(new MemChunkGroupDeletion())
-        .nextLevel(new MemChunkDeletion());
-    queryLevelProcessChain
-        .nextLevel(new MemTableGroupQuery())
-        .nextLevel(new MemTableQuery())
-        .nextLevel(new MemChunkGroupQuery())
-        .nextLevel(new MemChunkQuery());
+
+    List insertionLevelProcessClass = new ArrayList();
+    insertionLevelProcessClass.add(MemTableGroupInsertion.class.getName());
+    insertionLevelProcessClass.add(MemTableInsertion.class.getName());
+    insertionLevelProcessClass.add(MemChunkGroupInsertion.class.getName());
+    insertionLevelProcessClass.add(MemChunkInsertion.class.getName());
+
+    List deletionLevelProcessClass = new ArrayList<>();
+    deletionLevelProcessClass.add(MemTableGroupDeletion.class.getName());
+    deletionLevelProcessClass.add(MemTableDeletion.class.getName());
+    deletionLevelProcessClass.add(MemChunkGroupDeletion.class.getName());
+    deletionLevelProcessClass.add(MemChunkDeletion.class.getName());
+
+    List queryLevelProcessClass = new ArrayList<>();
+    queryLevelProcessClass.add(MemTableGroupQuery.class.getName());
+    queryLevelProcessClass.add(MemTableQuery.class.getName());
+    queryLevelProcessClass.add(MemChunkGroupQuery.class.getName());
+    queryLevelProcessClass.add(MemChunkQuery.class.getName());
+
+    Property property = new Property();
+    property.setDeletionLevelProcessClass(deletionLevelProcessClass);
+    property.setInsertionLevelProcessClass(insertionLevelProcessClass);
+    property.setQueryLevelProcessClass(queryLevelProcessClass);
+
     try {
       WALManager walManager =
           new WALManager(
@@ -101,21 +98,8 @@ public TagInvertedIndex(String schemaDirPath) {
               tagSchemaConfig.getWalBufferSize(),
               new WALEntry(),
               false);
-      InsertionManager insertionManager =
-          new InsertionManager<>(walManager);
-      DeletionManager deletionManager =
-          new DeletionManager<>(walManager);
-      QueryManager queryManager = new QueryManager<>();
-      RecoverManager> recoverManager = new RecoverManager<>(walManager);
-      insertionManager.setLevelProcessChain(insertionLevelProcessChain);
-      deletionManager.setLevelProcessChain(deletionLevelProcessChain);
-      queryManager.setLevelProcessChain(queryLevelProcessChain);
-      lsmEngine = new LSMEngine<>();
-      lsmEngine.setDeletionManager(deletionManager);
-      lsmEngine.setInsertionManager(insertionManager);
-      lsmEngine.setQueryManager(queryManager);
-      lsmEngine.setWalManager(walManager);
-      lsmEngine.setRecoverManager(recoverManager);
+      LSMEngineDirector lsmEngineDirector = new LSMEngineDirector<>();
+      lsmEngine = lsmEngineDirector.getLSMEngine(property, walManager);
       lsmEngine.setRootMemNode(new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable()));
       lsmEngine.recover();
     } catch (Exception e) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java
index 6df2b43057563..be7e3e9a6ae13 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java
@@ -35,6 +35,8 @@ public class MemTableGroup {
   // the largest device id saved by the current MemTable
   private int maxDeviceID;
 
+  public MemTableGroup() {}
+
   public MemTableGroup(int numOfDeviceIdsInMemTable) {
     this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable;
     workingMemTable = new MemTable(MemTable.WORKING);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
new file mode 100644
index 0000000000000..cb8fe17f6c8e5
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.engine;
+
+import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.manager.DeletionManager;
+import org.apache.iotdb.lsm.manager.InsertionManager;
+import org.apache.iotdb.lsm.manager.QueryManager;
+import org.apache.iotdb.lsm.manager.RecoverManager;
+import org.apache.iotdb.lsm.manager.WALManager;
+import org.apache.iotdb.lsm.request.DeletionRequest;
+import org.apache.iotdb.lsm.request.InsertionRequest;
+import org.apache.iotdb.lsm.request.QueryRequest;
+
+public class LSMEngineBuilder {
+
+  private LSMEngine lsmEngine;
+
+  public LSMEngineBuilder() {
+    lsmEngine = new LSMEngine<>();
+  }
+
+  public LSMEngineBuilder buildWalManager(WALManager walManager) {
+    lsmEngine.setWalManager(walManager);
+    return this;
+  }
+
+  public  LSMEngineBuilder buildInsertionManager(
+      LevelProcessChain levelProcessChain) {
+    InsertionManager insertionManager = new InsertionManager<>(lsmEngine.getWalManager());
+    insertionManager.setLevelProcessChain(levelProcessChain);
+    buildInsertionManager(insertionManager);
+    return this;
+  }
+
+  public  LSMEngineBuilder buildInsertionManager(
+      InsertionManager insertionManager) {
+    lsmEngine.setInsertionManager(insertionManager);
+    return this;
+  }
+
+  public  LSMEngineBuilder buildDeletionManager(
+      LevelProcessChain levelProcessChain) {
+    DeletionManager deletionManager = new DeletionManager<>(lsmEngine.getWalManager());
+    deletionManager.setLevelProcessChain(levelProcessChain);
+    buildDeletionManager(deletionManager);
+    return this;
+  }
+
+  public  LSMEngineBuilder buildDeletionManager(
+      DeletionManager deletionManager) {
+    lsmEngine.setDeletionManager(deletionManager);
+    return this;
+  }
+
+  public  LSMEngineBuilder buildQueryManager(
+      LevelProcessChain levelProcessChain) {
+    QueryManager queryManager = new QueryManager<>();
+    queryManager.setLevelProcessChain(levelProcessChain);
+    buildQueryManager(queryManager);
+    return this;
+  }
+
+  public  LSMEngineBuilder buildQueryManager(
+      QueryManager queryManager) {
+    lsmEngine.setQueryManager(queryManager);
+    return this;
+  }
+
+  public LSMEngineBuilder buildRecoverManager(RecoverManager> recoverManager) {
+    lsmEngine.setRecoverManager(recoverManager);
+    return this;
+  }
+
+  public LSMEngineBuilder buildRecoverManager() {
+    RecoverManager> recoverManager = new RecoverManager<>(lsmEngine.getWalManager());
+    lsmEngine.setRecoverManager(recoverManager);
+    return this;
+  }
+
+  public LSMEngineBuilder buildRootMemNode(T rootMemNode) {
+    lsmEngine.setRootMemNode(rootMemNode);
+    return this;
+  }
+
+  public LSMEngine builder() {
+    return lsmEngine;
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
new file mode 100644
index 0000000000000..f116347ca84e9
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.engine;
+
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
+import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.levelProcess.ILevelProcess;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.manager.WALManager;
+import org.apache.iotdb.lsm.property.Property;
+import org.apache.iotdb.lsm.request.InsertionRequest;
+import org.apache.iotdb.lsm.request.QueryRequest;
+import org.apache.iotdb.lsm.request.Request;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.List;
+
+public class LSMEngineDirector {
+  private static final Logger logger = LoggerFactory.getLogger(LSMEngineDirector.class);
+
+  LSMEngineBuilder lsmEngineBuilder;
+
+  public LSMEngineDirector(LSMEngineBuilder lsmEngineBuilder) {
+    this.lsmEngineBuilder = lsmEngineBuilder;
+  }
+
+  public LSMEngineDirector() {
+    lsmEngineBuilder = new LSMEngineBuilder<>();
+  }
+
+  public LSMEngine getLSMEngine(Property property, WALManager walManager) {
+    try {
+      LevelProcessChain insertionLevelProcessChain =
+          generateLevelProcessChain(property.getInsertionLevelProcessClass());
+      LevelProcessChain deletionLevelProcessChain =
+          generateLevelProcessChain(property.getDeletionLevelProcessClass());
+      LevelProcessChain queryLevelProcessChain =
+          generateLevelProcessChain(property.getQueryLevelProcessClass());
+
+      return lsmEngineBuilder
+          .buildWalManager(walManager)
+          .buildQueryManager(queryLevelProcessChain)
+          .buildInsertionManager(insertionLevelProcessChain)
+          .buildDeletionManager(deletionLevelProcessChain)
+          .buildRecoverManager()
+          .builder();
+
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+    return null;
+  }
+
+  private 
+      LevelProcessChain generateLevelProcessChain(List levelProcessClassNames) {
+    LevelProcessChain levelProcessChain = new LevelProcessChain<>();
+    try {
+      if (levelProcessClassNames.size() > 0) {
+        ILevelProcess iLevelProcess =
+            levelProcessChain.nextLevel(generateLevelProcess(levelProcessClassNames.get(0)));
+        for (int i = 1; i < levelProcessClassNames.size(); i++) {
+          iLevelProcess =
+              iLevelProcess.nextLevel(generateLevelProcess(levelProcessClassNames.get(i)));
+        }
+      }
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+    return levelProcessChain;
+  }
+
+  private 
+      ILevelProcess generateLevelProcess(String className)
+          throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException,
+              InstantiationException, IllegalAccessException {
+    Class c = Class.forName(className);
+    ILevelProcess result =
+        (ILevelProcess) c.getDeclaredConstructor().newInstance();
+    return result;
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/Property.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/Property.java
new file mode 100644
index 0000000000000..531ad0d245514
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/Property.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.property;
+
+import java.util.List;
+
+public class Property {
+
+  List insertionLevelProcessClass;
+
+  List deletionLevelProcessClass;
+
+  List queryLevelProcessClass;
+
+  public List getInsertionLevelProcessClass() {
+    return insertionLevelProcessClass;
+  }
+
+  public void setInsertionLevelProcessClass(List insertionLevelProcessClass) {
+    this.insertionLevelProcessClass = insertionLevelProcessClass;
+  }
+
+  public List getDeletionLevelProcessClass() {
+    return deletionLevelProcessClass;
+  }
+
+  public void setDeletionLevelProcessClass(List deletionLevelProcessClass) {
+    this.deletionLevelProcessClass = deletionLevelProcessClass;
+  }
+
+  public List getQueryLevelProcessClass() {
+    return queryLevelProcessClass;
+  }
+
+  public void setQueryLevelProcessClass(List queryLevelProcessClass) {
+    this.queryLevelProcessClass = queryLevelProcessClass;
+  }
+}

From 38b308094e79f39315503ff90ade6a36ed8393dc Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sun, 23 Oct 2022 17:12:31 +0800
Subject: [PATCH 42/53] rename WalRecord

---
 .../tagSchemaRegion/tagIndex/wal/WALEntry.java         |  8 ++++----
 .../tagSchemaRegion/tagIndex/wal/WALManager.java       |  4 ++--
 .../java/org/apache/iotdb/lsm/manager/WALManager.java  |  4 ++--
 .../main/java/org/apache/iotdb/lsm/wal/IWALReader.java |  2 +-
 .../iotdb/lsm/wal/{WALRecord.java => IWALRecord.java}  |  4 ++--
 .../main/java/org/apache/iotdb/lsm/wal/IWALWriter.java |  2 +-
 .../main/java/org/apache/iotdb/lsm/wal/WALReader.java  | 10 +++++-----
 .../main/java/org/apache/iotdb/lsm/wal/WALWriter.java  |  2 +-
 8 files changed, 18 insertions(+), 18 deletions(-)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/{WALRecord.java => IWALRecord.java} (94%)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java
index 1b769a2ab7529..d3d8176dfb329 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal;
 
-import org.apache.iotdb.lsm.wal.WALRecord;
+import org.apache.iotdb.lsm.wal.IWALRecord;
 import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
 
 import java.io.DataInputStream;
@@ -28,7 +28,7 @@
 import java.util.List;
 
 /** represents a record in the wal file */
-public class WALEntry implements WALRecord {
+public class WALEntry implements IWALRecord {
 
   // can be insertion(1) or deletion(2)
   private int type;
@@ -89,9 +89,9 @@ public void deserialize(DataInputStream stream) throws IOException {
    * @return wal record
    */
   @Override
-  public WALRecord clone() {
+  public IWALRecord clone() {
     try {
-      return (WALRecord) super.clone();
+      return (IWALRecord) super.clone();
     } catch (CloneNotSupportedException e) {
       throw new AssertionError(e.getMessage());
     }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
index bf79ea903853e..4f9011af423e9 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
@@ -21,8 +21,8 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.wal.IWALRecord;
 import org.apache.iotdb.lsm.wal.WALReader;
-import org.apache.iotdb.lsm.wal.WALRecord;
 
 import java.io.IOException;
 
@@ -36,7 +36,7 @@ public WALManager(
       String schemaDirPath,
       String walFileName,
       int walBufferSize,
-      WALRecord walRecord,
+      IWALRecord walRecord,
       boolean forceEachWrite)
       throws IOException {
     super(schemaDirPath, walFileName, walBufferSize, walRecord, forceEachWrite);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
index c5c8ba42651e6..8d251a2189564 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
@@ -19,8 +19,8 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.wal.IWALRecord;
 import org.apache.iotdb.lsm.wal.WALReader;
-import org.apache.iotdb.lsm.wal.WALRecord;
 import org.apache.iotdb.lsm.wal.WALWriter;
 
 import java.io.File;
@@ -49,7 +49,7 @@ public WALManager(
       String schemaDirPath,
       String walFileName,
       int walBufferSize,
-      WALRecord walRecord,
+      IWALRecord walRecord,
       boolean forceEachWrite)
       throws IOException {
     this.schemaDirPath = schemaDirPath;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java
index c1e42f74113b4..5309737fb6957 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java
@@ -44,5 +44,5 @@ public interface IWALReader {
    *
    * @throws FileNotFoundException
    */
-  WALRecord next() throws FileNotFoundException;
+  IWALRecord next() throws FileNotFoundException;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALRecord.java
similarity index 94%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALRecord.java
index 1eef0577f454d..18e48fb325165 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALRecord.java
@@ -24,7 +24,7 @@
 import java.util.List;
 
 /** represents a wal record, which can be extended to implement more complex wal records */
-public interface WALRecord extends Cloneable {
+public interface IWALRecord extends Cloneable {
 
   /**
    * serialize the wal record
@@ -42,7 +42,7 @@ public interface WALRecord extends Cloneable {
   void deserialize(DataInputStream stream) throws IOException;
 
   // generate wal record using prototyping pattern
-  WALRecord clone();
+  IWALRecord clone();
 
   List getKeys();
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java
index 3a7b4e2cda854..4a41b5aa212b0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java
@@ -29,7 +29,7 @@ public interface IWALWriter {
    * @param walRecord record to be written
    * @throws IOException
    */
-  void write(WALRecord walRecord) throws IOException;
+  void write(IWALRecord walRecord) throws IOException;
 
   /**
    * force brush
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java
index a5a70b62f1722..63469afdda2d3 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java
@@ -35,13 +35,13 @@ public class WALReader implements IWALReader {
   // wal file
   private final File logFile;
   // wal record prototype, clone on read
-  private final WALRecord prototype;
+  private final IWALRecord prototype;
   private DataInputStream logStream;
   // next wal record
-  private WALRecord nextRecord;
+  private IWALRecord nextRecord;
   private boolean fileCorrupted = false;
 
-  public WALReader(File logFile, WALRecord prototype) throws IOException {
+  public WALReader(File logFile, IWALRecord prototype) throws IOException {
     this.logFile = logFile;
     this.logStream =
         new DataInputStream(new BufferedInputStream(Files.newInputStream(logFile.toPath())));
@@ -83,11 +83,11 @@ public boolean hasNext() {
   }
 
   @Override
-  public WALRecord next() {
+  public IWALRecord next() {
     if (nextRecord == null) {
       throw new NoSuchElementException();
     }
-    WALRecord walRecord = nextRecord;
+    IWALRecord walRecord = nextRecord;
     nextRecord = null;
     return walRecord;
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java
index 5ce97779ce129..9b3924c01586e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java
@@ -59,7 +59,7 @@ public WALWriter(File logFile, int walBufferSize, boolean forceEachWrite)
    * @throws IOException
    */
   @Override
-  public void write(WALRecord walRecord) throws IOException {
+  public void write(IWALRecord walRecord) throws IOException {
     if (channel == null) {
       fileOutputStream = new FileOutputStream(logFile, true);
       channel = fileOutputStream.getChannel();

From e824d2c50eeab7b6569833cf74f59f29fe7dbfdb Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Sun, 23 Oct 2022 23:19:30 +0800
Subject: [PATCH 43/53] add annotation

---
 schema-engine-tag/pom.xml                     |   5 +
 .../tagIndex/TagInvertedIndex.java            |  41 +------
 .../tagIndex/deletion/MemChunkDeletion.java   |   2 +
 .../deletion/MemChunkGroupDeletion.java       |   2 +
 .../tagIndex/deletion/MemTableDeletion.java   |   2 +
 .../deletion/MemTableGroupDeletion.java       |   2 +
 .../insertion/MemChunkGroupInsertion.java     |   2 +
 .../tagIndex/insertion/MemChunkInsertion.java |   2 +
 .../insertion/MemTableGroupInsertion.java     |   2 +
 .../tagIndex/insertion/MemTableInsertion.java |   2 +
 .../tagIndex/query/MemChunkGroupQuery.java    |   2 +
 .../tagIndex/query/MemChunkQuery.java         |   2 +
 .../tagIndex/query/MemTableGroupQuery.java    |   2 +
 .../tagIndex/query/MemTableQuery.java         |   2 +
 .../iotdb/lsm/annotation/DeletionProcess.java |  31 ++++++
 .../lsm/annotation/InsertionProcess.java      |  31 ++++++
 .../iotdb/lsm/annotation/QueryProcess.java    |  31 ++++++
 .../iotdb/lsm/engine/LSMEngineDirector.java   |  11 ++
 .../lsm/property/PropertyDescriptor.java      | 104 ++++++++++++++++++
 19 files changed, 240 insertions(+), 38 deletions(-)
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcess.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcess.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcess.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java

diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml
index 04ea391c9285a..594f702214085 100644
--- a/schema-engine-tag/pom.xml
+++ b/schema-engine-tag/pom.xml
@@ -40,6 +40,11 @@
             ${project.version}
             provided
         
+        
+            org.reflections
+            reflections
+            0.10.2
+        
     
     
         schema-engine-tag
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index 4f7ef1a72d103..49f062f9fe40e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -24,24 +24,11 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemChunkGroupDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.MemTableGroupDeletion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkGroupInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemChunkInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableGroupInsertion;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.MemTableInsertion;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkGroupQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemChunkQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableGroupQuery;
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.MemTableQuery;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
 import org.apache.iotdb.lsm.engine.LSMEngine;
 import org.apache.iotdb.lsm.engine.LSMEngineDirector;
-import org.apache.iotdb.lsm.property.Property;
 
 import org.roaringbitmap.RoaringBitmap;
 import org.slf4j.Logger;
@@ -66,30 +53,6 @@ public class TagInvertedIndex implements ITagInvertedIndex {
   LSMEngine lsmEngine;
 
   public TagInvertedIndex(String schemaDirPath) {
-
-    List insertionLevelProcessClass = new ArrayList();
-    insertionLevelProcessClass.add(MemTableGroupInsertion.class.getName());
-    insertionLevelProcessClass.add(MemTableInsertion.class.getName());
-    insertionLevelProcessClass.add(MemChunkGroupInsertion.class.getName());
-    insertionLevelProcessClass.add(MemChunkInsertion.class.getName());
-
-    List deletionLevelProcessClass = new ArrayList<>();
-    deletionLevelProcessClass.add(MemTableGroupDeletion.class.getName());
-    deletionLevelProcessClass.add(MemTableDeletion.class.getName());
-    deletionLevelProcessClass.add(MemChunkGroupDeletion.class.getName());
-    deletionLevelProcessClass.add(MemChunkDeletion.class.getName());
-
-    List queryLevelProcessClass = new ArrayList<>();
-    queryLevelProcessClass.add(MemTableGroupQuery.class.getName());
-    queryLevelProcessClass.add(MemTableQuery.class.getName());
-    queryLevelProcessClass.add(MemChunkGroupQuery.class.getName());
-    queryLevelProcessClass.add(MemChunkQuery.class.getName());
-
-    Property property = new Property();
-    property.setDeletionLevelProcessClass(deletionLevelProcessClass);
-    property.setInsertionLevelProcessClass(insertionLevelProcessClass);
-    property.setQueryLevelProcessClass(queryLevelProcessClass);
-
     try {
       WALManager walManager =
           new WALManager(
@@ -99,7 +62,9 @@ public TagInvertedIndex(String schemaDirPath) {
               new WALEntry(),
               false);
       LSMEngineDirector lsmEngineDirector = new LSMEngineDirector<>();
-      lsmEngine = lsmEngineDirector.getLSMEngine(property, walManager);
+      lsmEngine =
+          lsmEngineDirector.getLSMEngine(
+              "org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex", walManager);
       lsmEngine.setRootMemNode(new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable()));
       lsmEngine.recover();
     } catch (Exception e) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
index 8996f1e3f0ad0..52c5e69e52404 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
@@ -20,12 +20,14 @@
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
+import org.apache.iotdb.lsm.annotation.DeletionProcess;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
 
 import java.util.List;
 
 /** deletion for MemChunk */
+@DeletionProcess(level = 3)
 public class MemChunkDeletion extends DeleteLevelProcess {
 
   /**
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
index 8dd06a208ba56..42451293a2e81 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
@@ -21,6 +21,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
+import org.apache.iotdb.lsm.annotation.DeletionProcess;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
 
@@ -28,6 +29,7 @@
 import java.util.List;
 
 /** deletion for MemChunkGroup */
+@DeletionProcess(level = 2)
 public class MemChunkGroupDeletion
     extends DeleteLevelProcess {
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
index eff0c21b8c439..91f9d45c03d2d 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
@@ -21,6 +21,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
+import org.apache.iotdb.lsm.annotation.DeletionProcess;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
 
@@ -29,6 +30,7 @@
 import java.util.Set;
 
 /** deletion for MemTable */
+@DeletionProcess(level = 1)
 public class MemTableDeletion extends DeleteLevelProcess {
 
   /**
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
index 7091f233c1b92..7094681a5baf5 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
@@ -21,12 +21,14 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.lsm.annotation.DeletionProcess;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
 
 import java.util.ArrayList;
 import java.util.List;
 
+@DeletionProcess(level = 0)
 public class MemTableGroupDeletion
     extends DeleteLevelProcess {
   @Override
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
index a07f196324bd2..66bb854ec12e4 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
@@ -21,6 +21,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
+import org.apache.iotdb.lsm.annotation.InsertionProcess;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
 
@@ -28,6 +29,7 @@
 import java.util.List;
 
 /** insertion for MemChunkGroup */
+@InsertionProcess(level = 2)
 public class MemChunkGroupInsertion
     extends InsertLevelProcess {
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
index 9db253d790d5a..e5df48d19fe97 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
@@ -20,12 +20,14 @@
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
+import org.apache.iotdb.lsm.annotation.InsertionProcess;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
 
 import java.util.List;
 
 /** insertion for MemChunk */
+@InsertionProcess(level = 3)
 public class MemChunkInsertion extends InsertLevelProcess {
 
   /**
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
index a4b8fd4adae00..b5d57aae7aed1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
@@ -22,6 +22,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.lsm.annotation.InsertionProcess;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
 
@@ -29,6 +30,7 @@
 import java.util.List;
 import java.util.Map;
 
+@InsertionProcess(level = 0)
 public class MemTableGroupInsertion
     extends InsertLevelProcess {
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
index 7a35b6d65763a..f0faf7bad1e71 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
@@ -21,6 +21,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
+import org.apache.iotdb.lsm.annotation.InsertionProcess;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
 
@@ -28,6 +29,7 @@
 import java.util.List;
 
 /** insertion for MemTable */
+@InsertionProcess(level = 1)
 public class MemTableInsertion
     extends InsertLevelProcess {
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
index 51bf85c30379e..7d355db1f9d3f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
@@ -21,6 +21,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
+import org.apache.iotdb.lsm.annotation.QueryProcess;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
 
@@ -28,6 +29,7 @@
 import java.util.List;
 
 /** query for MemChunkGroup */
+@QueryProcess(level = 2)
 public class MemChunkGroupQuery extends QueryLevelProcess {
 
   /**
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
index f94ea6d8f43b0..aaea71f5eb1c5 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
@@ -20,6 +20,7 @@
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
+import org.apache.iotdb.lsm.annotation.QueryProcess;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
 
@@ -28,6 +29,7 @@
 import java.util.List;
 
 /** query for MemChunk */
+@QueryProcess(level = 3)
 public class MemChunkQuery extends QueryLevelProcess {
 
   /**
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
index 03efb4bca85d1..f24c7decc19cf 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
@@ -21,12 +21,14 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.lsm.annotation.QueryProcess;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
 
 import java.util.ArrayList;
 import java.util.List;
 
+@QueryProcess(level = 0)
 public class MemTableGroupQuery extends QueryLevelProcess {
   @Override
   public List getChildren(
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
index 3ae18f148af14..d27dfba3347bf 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
@@ -21,6 +21,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
+import org.apache.iotdb.lsm.annotation.QueryProcess;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
 
@@ -31,6 +32,7 @@
 import java.util.Set;
 
 /** query for MemTable */
+@QueryProcess(level = 1)
 public class MemTableQuery extends QueryLevelProcess {
 
   /**
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcess.java
new file mode 100644
index 0000000000000..847b42f421cd5
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcess.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface DeletionProcess {
+  // level of the levelProcess
+  int level() default -1;
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcess.java
new file mode 100644
index 0000000000000..9af1c450450ff
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcess.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface InsertionProcess {
+  // level of the levelProcess
+  int level() default -1;
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcess.java
new file mode 100644
index 0000000000000..4a712c082130e
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcess.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface QueryProcess {
+  // level of the levelProcess
+  int level() default -1;
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
index f116347ca84e9..6ca9b36b3bb54 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
@@ -27,6 +27,7 @@
 import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
 import org.apache.iotdb.lsm.manager.WALManager;
 import org.apache.iotdb.lsm.property.Property;
+import org.apache.iotdb.lsm.property.PropertyDescriptor;
 import org.apache.iotdb.lsm.request.InsertionRequest;
 import org.apache.iotdb.lsm.request.QueryRequest;
 import org.apache.iotdb.lsm.request.Request;
@@ -73,6 +74,16 @@ public LSMEngine getLSMEngine(Property property, WALManager walManager) {
     return null;
   }
 
+  public LSMEngine getLSMEngine(String packageName, WALManager walManager) {
+    try {
+      Property property = PropertyDescriptor.getProperty(packageName);
+      return getLSMEngine(property, walManager);
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+    return null;
+  }
+
   private 
       LevelProcessChain generateLevelProcessChain(List levelProcessClassNames) {
     LevelProcessChain levelProcessChain = new LevelProcessChain<>();
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
new file mode 100644
index 0000000000000..c245a1dfa40cf
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.property;
+
+import org.apache.iotdb.lsm.annotation.DeletionProcess;
+import org.apache.iotdb.lsm.annotation.InsertionProcess;
+import org.apache.iotdb.lsm.annotation.QueryProcess;
+
+import org.reflections.Reflections;
+import org.reflections.util.ConfigurationBuilder;
+import org.reflections.util.FilterBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+public class PropertyDescriptor {
+
+  public static Property getProperty(String packageName) throws Exception {
+    Reflections reflections =
+        new Reflections(
+            new ConfigurationBuilder()
+                .forPackage(packageName)
+                .filterInputsBy(new FilterBuilder().includePackage(packageName)));
+    Property property = new Property();
+    setDeletionLevelProcess(property, reflections);
+    setInsertionLevelProcess(property, reflections);
+    setQueryLevelProcess(property, reflections);
+    return property;
+  }
+
+  private static void setInsertionLevelProcess(Property property, Reflections reflections)
+      throws Exception {
+    Set> annotated = reflections.getTypesAnnotatedWith(InsertionProcess.class);
+    List levelProcessClass = new ArrayList<>();
+    for (Class clz : annotated) {
+      InsertionProcess annotationInfo = clz.getAnnotation(InsertionProcess.class);
+      int level = annotationInfo.level();
+      if (level < levelProcessClass.size()) {
+        levelProcessClass.set(level, clz.getName());
+      } else {
+        for (int i = levelProcessClass.size(); i < level; i++) {
+          levelProcessClass.add("");
+        }
+        levelProcessClass.add(clz.getName());
+      }
+    }
+    property.setInsertionLevelProcessClass(levelProcessClass);
+  }
+
+  private static void setDeletionLevelProcess(Property property, Reflections reflections)
+      throws Exception {
+    Set> annotated = reflections.getTypesAnnotatedWith(DeletionProcess.class);
+    List levelProcessClass = new ArrayList<>();
+    for (Class clz : annotated) {
+      DeletionProcess annotationInfo = clz.getAnnotation(DeletionProcess.class);
+      int level = annotationInfo.level();
+      if (level < levelProcessClass.size()) {
+        levelProcessClass.set(level, clz.getName());
+      } else {
+        for (int i = levelProcessClass.size(); i < level; i++) {
+          levelProcessClass.add("");
+        }
+        levelProcessClass.add(clz.getName());
+      }
+    }
+    property.setDeletionLevelProcessClass(levelProcessClass);
+  }
+
+  private static void setQueryLevelProcess(Property property, Reflections reflections)
+      throws Exception {
+    List levelProcessClass = new ArrayList<>();
+    Set> annotated = reflections.getTypesAnnotatedWith(QueryProcess.class);
+    for (Class clz : annotated) {
+      QueryProcess annotationInfo = clz.getAnnotation(QueryProcess.class);
+      int level = annotationInfo.level();
+      if (level < levelProcessClass.size()) {
+        levelProcessClass.set(level, clz.getName());
+      } else {
+        for (int i = levelProcessClass.size(); i < level; i++) {
+          levelProcessClass.add("");
+        }
+        levelProcessClass.add(clz.getName());
+      }
+    }
+    property.setQueryLevelProcessClass(levelProcessClass);
+  }
+}

From 7230b58f197e4f10a459254cc1abd1dd59ed962d Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Mon, 24 Oct 2022 10:37:08 +0800
Subject: [PATCH 44/53] iotdb-datanode.properties

---
 .../resources/conf/iotdb-datanode.properties      | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties
index 5a88b1f8f635d..73150bd4bef5d 100644
--- a/server/src/assembly/resources/conf/iotdb-datanode.properties
+++ b/server/src/assembly/resources/conf/iotdb-datanode.properties
@@ -1070,23 +1070,10 @@ timestamp_precision=ms
 # Datatype: float
 # group_by_fill_cache_size_in_mb=1.0
 
-####################
-### IDTable Configuration
-####################
-# Datatype: boolean
-# enable_id_table=false
-
-# Datatype: boolean
-# enable_id_table_log_file=false
-
-# Choose the device id transformation method. The value could be Plain and SHA256. If the provided value doesn't match any pre-defined value, Plain will be used as default.
-# Datatype: string
-# device_id_transformation_method=Plain
-
 ####################
 ### Schema Engine Configuration
 ####################
-# Choose the mode of schema engine. The value could be Memory,Schema_File,Rocksdb_based and Tag. If the provided value doesn't match any pre-defined value, Memory mode will be used as default.
+# Choose the mode of schema engine. The value could be Memory,Schema_File and Rocksdb_based. If the provided value doesn't match any pre-defined value, Memory mode will be used as default.
 # Datatype: string
 # schema_engine_mode=Memory
 

From 47dbc111f349e45968aec7c31818ebc4d805212e Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Mon, 24 Oct 2022 11:06:42 +0800
Subject: [PATCH 45/53] make Request as interface

---
 .../tagIndex/Request/DeletionRequest.java     |  4 +-
 .../tagIndex/Request/InsertionRequest.java    |  4 +-
 .../tagIndex/Request/QueryRequest.java        |  3 +-
 .../tagIndex/wal/WALManager.java              |  6 +-
 .../apache/iotdb/lsm/engine/ILSMEngine.java   | 18 ++---
 .../apache/iotdb/lsm/engine/LSMEngine.java    | 81 ++++++-------------
 .../iotdb/lsm/engine/LSMEngineBuilder.java    | 20 ++---
 .../iotdb/lsm/engine/LSMEngineDirector.java   | 16 ++--
 .../lsm/levelProcess/LevelProcessChain.java   |  4 +-
 .../iotdb/lsm/manager/BasicLSMManager.java    |  4 +-
 .../iotdb/lsm/manager/DeletionManager.java    |  4 +-
 .../iotdb/lsm/manager/InsertionManager.java   |  4 +-
 .../iotdb/lsm/manager/QueryManager.java       |  4 +-
 .../iotdb/lsm/manager/RecoverManager.java     |  4 +-
 .../apache/iotdb/lsm/manager/WALManager.java  |  6 +-
 .../iotdb/lsm/recover/IRecoverable.java       |  4 +-
 ...ionRequest.java => IDeletionIRequest.java} | 13 +--
 ...ryRequest.java => IInsertionIRequest.java} | 11 ++-
 ...letionRequest.java => IQueryIRequest.java} | 11 +--
 .../request/{Request.java => IRequest.java}   | 24 ++----
 .../apache/iotdb/lsm/request/RequestType.java |  3 +-
 21 files changed, 108 insertions(+), 140 deletions(-)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/{InsertionRequest.java => IDeletionIRequest.java} (80%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/{QueryRequest.java => IInsertionIRequest.java} (79%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/{DeletionRequest.java => IQueryIRequest.java} (81%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/{Request.java => IRequest.java} (66%)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
index bc9f24a899039..601220ad64a0a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
@@ -19,11 +19,11 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
 import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.request.IDeletionIRequest;
 
 import java.util.List;
 
-public class DeletionRequest
-    extends org.apache.iotdb.lsm.request.DeletionRequest {
+public class DeletionRequest implements IDeletionIRequest {
 
   List keys;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
index a475f553ece08..716fb9909c0e0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
@@ -19,11 +19,11 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
 import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.request.IInsertionIRequest;
 
 import java.util.List;
 
-public class InsertionRequest
-    extends org.apache.iotdb.lsm.request.InsertionRequest {
+public class InsertionRequest implements IInsertionIRequest {
 
   List keys;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
index c62d0db758041..acb48036db03c 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
@@ -19,12 +19,13 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
 import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.request.IQueryIRequest;
 
 import org.roaringbitmap.RoaringBitmap;
 
 import java.util.List;
 
-public class QueryRequest extends org.apache.iotdb.lsm.request.QueryRequest {
+public class QueryRequest implements IQueryIRequest {
 
   List keys;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
index 4f9011af423e9..59549b2d07fb9 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
@@ -20,7 +20,7 @@
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
-import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.request.IRequest;
 import org.apache.iotdb.lsm.wal.IWALRecord;
 import org.apache.iotdb.lsm.wal.WALReader;
 
@@ -53,7 +53,7 @@ public WALManager(String schemaDirPath) {
    * @throws IOException
    */
   @Override
-  public synchronized void write(Request request) throws IOException {
+  public synchronized void write(IRequest request) throws IOException {
     if (isRecover()) return;
     switch (request.getRequestType()) {
       case INSERT:
@@ -73,7 +73,7 @@ public synchronized void write(Request request) throws IOException {
    * @return request context
    */
   @Override
-  public synchronized Request read() {
+  public synchronized IRequest read() {
     WALReader walReader = getWalReader();
     if (walReader.hasNext()) {
       WALEntry walEntry = (WALEntry) getWalReader().next();
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
index 6d26e77f42598..33860c644f243 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
@@ -21,29 +21,29 @@
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.request.DeletionRequest;
-import org.apache.iotdb.lsm.request.InsertionRequest;
-import org.apache.iotdb.lsm.request.QueryRequest;
+import org.apache.iotdb.lsm.request.IDeletionIRequest;
+import org.apache.iotdb.lsm.request.IInsertionIRequest;
+import org.apache.iotdb.lsm.request.IQueryIRequest;
 
 import java.io.IOException;
 
 public interface ILSMEngine {
 
-   void insert(InsertionRequest insertionRequest) throws Exception;
+   void insert(IInsertionIRequest insertionRequest) throws Exception;
 
    void insert(
-      InsertionRequest insertionRequest, InsertRequestContext insertRequestContext)
+      IInsertionIRequest insertionRequest, InsertRequestContext insertRequestContext)
       throws Exception;
 
-   void query(QueryRequest queryRequest) throws Exception;
+   void query(IQueryIRequest queryRequest) throws Exception;
 
-   void query(QueryRequest queryRequest, QueryRequestContext queryRequestContext)
+   void query(IQueryIRequest queryRequest, QueryRequestContext queryRequestContext)
       throws Exception;
 
-   void delete(DeletionRequest deletionRequest) throws Exception;
+   void delete(IDeletionIRequest deletionRequest) throws Exception;
 
    void delete(
-      DeletionRequest deletionRequest, DeleteRequestContext deleteRequestContext)
+      IDeletionIRequest deletionRequest, DeleteRequestContext deleteRequestContext)
       throws Exception;
 
   void recover() throws Exception;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
index 7346753dc1194..de761df03166a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
@@ -27,20 +27,20 @@
 import org.apache.iotdb.lsm.manager.RecoverManager;
 import org.apache.iotdb.lsm.manager.WALManager;
 import org.apache.iotdb.lsm.recover.IRecoverable;
-import org.apache.iotdb.lsm.request.DeletionRequest;
-import org.apache.iotdb.lsm.request.InsertionRequest;
-import org.apache.iotdb.lsm.request.QueryRequest;
-import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.request.IDeletionIRequest;
+import org.apache.iotdb.lsm.request.IInsertionIRequest;
+import org.apache.iotdb.lsm.request.IQueryIRequest;
+import org.apache.iotdb.lsm.request.IRequest;
 
 import java.io.IOException;
 
 public class LSMEngine implements ILSMEngine, IRecoverable {
 
-  private InsertionManager insertionManager;
+  private InsertionManager insertionManager;
 
-  private DeletionManager deletionManager;
+  private DeletionManager deletionManager;
 
-  private QueryManager queryManager;
+  private QueryManager queryManager;
 
   private WALManager walManager;
 
@@ -50,54 +50,41 @@ public class LSMEngine implements ILSMEngine, IRecoverable {
 
   public LSMEngine() {}
 
-  public LSMEngine(WALManager walManager) throws Exception {
-    this.walManager = walManager;
-    insertionManager = new InsertionManager<>(walManager);
-    deletionManager = new DeletionManager<>(walManager);
-    queryManager = new QueryManager<>();
-    recoverManager = new RecoverManager<>(walManager);
-    recoverManager.recover(this);
-  }
-
-  public T getRootMemNode() {
-    return rootMemNode;
-  }
-
   public void setRootMemNode(T rootMemNode) {
     this.rootMemNode = rootMemNode;
   }
 
   @Override
-  public  void insert(InsertionRequest insertionRequest) throws Exception {
+  public  void insert(IInsertionIRequest insertionRequest) throws Exception {
     insertionManager.process(rootMemNode, insertionRequest, new InsertRequestContext());
   }
 
   @Override
   public  void insert(
-      InsertionRequest insertionRequest, InsertRequestContext insertRequestContext)
+      IInsertionIRequest insertionRequest, InsertRequestContext insertRequestContext)
       throws Exception {
     insertionManager.process(rootMemNode, insertionRequest, insertRequestContext);
   }
 
   @Override
-  public  void query(QueryRequest queryRequest) throws Exception {
+  public  void query(IQueryIRequest queryRequest) throws Exception {
     queryManager.process(rootMemNode, queryRequest, new QueryRequestContext());
   }
 
   @Override
-  public  void query(QueryRequest queryRequest, QueryRequestContext queryRequestContext)
-      throws Exception {
+  public  void query(
+      IQueryIRequest queryRequest, QueryRequestContext queryRequestContext) throws Exception {
     queryManager.process(rootMemNode, queryRequest, queryRequestContext);
   }
 
   @Override
-  public  void delete(DeletionRequest deletionRequest) throws Exception {
+  public  void delete(IDeletionIRequest deletionRequest) throws Exception {
     deletionManager.process(rootMemNode, deletionRequest, new DeleteRequestContext());
   }
 
   @Override
   public  void delete(
-      DeletionRequest deletionRequest, DeleteRequestContext deleteRequestContext)
+      IDeletionIRequest deletionRequest, DeleteRequestContext deleteRequestContext)
       throws Exception {
     deletionManager.process(rootMemNode, deletionRequest, deleteRequestContext);
   }
@@ -112,56 +99,40 @@ public void clear() throws IOException {
     walManager.close();
   }
 
-  public InsertionManager getInsertionManager() {
-    return insertionManager;
-  }
-
-  public  void setInsertionManager(
+  protected  void setInsertionManager(
       InsertionManager insertionManager) {
-    this.insertionManager = (InsertionManager) insertionManager;
-  }
-
-  public DeletionManager getDeletionManager() {
-    return deletionManager;
+    this.insertionManager = (InsertionManager) insertionManager;
   }
 
-  public  void setDeletionManager(
+  protected  void setDeletionManager(
       DeletionManager deletionManager) {
-    this.deletionManager = (DeletionManager) deletionManager;
+    this.deletionManager = (DeletionManager) deletionManager;
   }
 
-  public QueryManager getQueryManager() {
-    return queryManager;
+  protected  void setQueryManager(QueryManager queryManager) {
+    this.queryManager = (QueryManager) queryManager;
   }
 
-  public  void setQueryManager(QueryManager queryManager) {
-    this.queryManager = (QueryManager) queryManager;
-  }
-
-  public WALManager getWalManager() {
+  protected WALManager getWalManager() {
     return walManager;
   }
 
-  public void setWalManager(WALManager walManager) {
+  protected void setWalManager(WALManager walManager) {
     this.walManager = walManager;
   }
 
-  public RecoverManager> getRecoverManager() {
-    return recoverManager;
-  }
-
-  public void setRecoverManager(RecoverManager> recoverManager) {
+  protected void setRecoverManager(RecoverManager> recoverManager) {
     this.recoverManager = recoverManager;
   }
 
   @Override
-  public  void recover(Request request) throws Exception {
+  public  void recover(IRequest request) throws Exception {
     switch (request.getRequestType()) {
       case INSERT:
-        insert((InsertionRequest) request);
+        insert((IInsertionIRequest) request);
         break;
       case DELETE:
-        delete((DeletionRequest) request);
+        delete((IDeletionIRequest) request);
         break;
       default:
         break;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
index cb8fe17f6c8e5..29c079b0ba205 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
@@ -27,9 +27,9 @@
 import org.apache.iotdb.lsm.manager.QueryManager;
 import org.apache.iotdb.lsm.manager.RecoverManager;
 import org.apache.iotdb.lsm.manager.WALManager;
-import org.apache.iotdb.lsm.request.DeletionRequest;
-import org.apache.iotdb.lsm.request.InsertionRequest;
-import org.apache.iotdb.lsm.request.QueryRequest;
+import org.apache.iotdb.lsm.request.IDeletionIRequest;
+import org.apache.iotdb.lsm.request.IInsertionIRequest;
+import org.apache.iotdb.lsm.request.IQueryIRequest;
 
 public class LSMEngineBuilder {
 
@@ -44,7 +44,7 @@ public LSMEngineBuilder buildWalManager(WALManager walManager) {
     return this;
   }
 
-  public  LSMEngineBuilder buildInsertionManager(
+  public  LSMEngineBuilder buildInsertionManager(
       LevelProcessChain levelProcessChain) {
     InsertionManager insertionManager = new InsertionManager<>(lsmEngine.getWalManager());
     insertionManager.setLevelProcessChain(levelProcessChain);
@@ -52,13 +52,13 @@ public  LSMEngineBuilder buildInsertionManager(
     return this;
   }
 
-  public  LSMEngineBuilder buildInsertionManager(
+  public  LSMEngineBuilder buildInsertionManager(
       InsertionManager insertionManager) {
     lsmEngine.setInsertionManager(insertionManager);
     return this;
   }
 
-  public  LSMEngineBuilder buildDeletionManager(
+  public  LSMEngineBuilder buildDeletionManager(
       LevelProcessChain levelProcessChain) {
     DeletionManager deletionManager = new DeletionManager<>(lsmEngine.getWalManager());
     deletionManager.setLevelProcessChain(levelProcessChain);
@@ -66,13 +66,13 @@ public  LSMEngineBuilder buildDeletionManager(
     return this;
   }
 
-  public  LSMEngineBuilder buildDeletionManager(
+  public  LSMEngineBuilder buildDeletionManager(
       DeletionManager deletionManager) {
     lsmEngine.setDeletionManager(deletionManager);
     return this;
   }
 
-  public  LSMEngineBuilder buildQueryManager(
+  public  LSMEngineBuilder buildQueryManager(
       LevelProcessChain levelProcessChain) {
     QueryManager queryManager = new QueryManager<>();
     queryManager.setLevelProcessChain(levelProcessChain);
@@ -80,7 +80,7 @@ public  LSMEngineBuilder buildQueryManager(
     return this;
   }
 
-  public  LSMEngineBuilder buildQueryManager(
+  public  LSMEngineBuilder buildQueryManager(
       QueryManager queryManager) {
     lsmEngine.setQueryManager(queryManager);
     return this;
@@ -102,7 +102,7 @@ public LSMEngineBuilder buildRootMemNode(T rootMemNode) {
     return this;
   }
 
-  public LSMEngine builder() {
+  public LSMEngine build() {
     return lsmEngine;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
index 6ca9b36b3bb54..8b0bb76f2265a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
@@ -28,9 +28,9 @@
 import org.apache.iotdb.lsm.manager.WALManager;
 import org.apache.iotdb.lsm.property.Property;
 import org.apache.iotdb.lsm.property.PropertyDescriptor;
-import org.apache.iotdb.lsm.request.InsertionRequest;
-import org.apache.iotdb.lsm.request.QueryRequest;
-import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.request.IInsertionIRequest;
+import org.apache.iotdb.lsm.request.IQueryIRequest;
+import org.apache.iotdb.lsm.request.IRequest;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -53,11 +53,11 @@ public LSMEngineDirector() {
 
   public LSMEngine getLSMEngine(Property property, WALManager walManager) {
     try {
-      LevelProcessChain insertionLevelProcessChain =
+      LevelProcessChain insertionLevelProcessChain =
           generateLevelProcessChain(property.getInsertionLevelProcessClass());
       LevelProcessChain deletionLevelProcessChain =
           generateLevelProcessChain(property.getDeletionLevelProcessClass());
-      LevelProcessChain queryLevelProcessChain =
+      LevelProcessChain queryLevelProcessChain =
           generateLevelProcessChain(property.getQueryLevelProcessClass());
 
       return lsmEngineBuilder
@@ -66,7 +66,7 @@ public LSMEngine getLSMEngine(Property property, WALManager walManager) {
           .buildInsertionManager(insertionLevelProcessChain)
           .buildDeletionManager(deletionLevelProcessChain)
           .buildRecoverManager()
-          .builder();
+          .build();
 
     } catch (Exception e) {
       logger.error(e.getMessage());
@@ -84,7 +84,7 @@ public LSMEngine getLSMEngine(String packageName, WALManager walManager) {
     return null;
   }
 
-  private 
+  private 
       LevelProcessChain generateLevelProcessChain(List levelProcessClassNames) {
     LevelProcessChain levelProcessChain = new LevelProcessChain<>();
     try {
@@ -102,7 +102,7 @@ LevelProcessChain generateLevelProcessChain(List levelProcessCl
     return levelProcessChain;
   }
 
-  private 
+  private 
       ILevelProcess generateLevelProcess(String className)
           throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException,
               InstantiationException, IllegalAccessException {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
index 8c2dae4713c44..9f89bda3377b2 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
@@ -19,9 +19,9 @@
 package org.apache.iotdb.lsm.levelProcess;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.request.IRequest;
 
-public class LevelProcessChain {
+public class LevelProcessChain {
 
   // the level process of the first layer of memory nodes
   ILevelProcess headLevelProcess;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
index 47ec4bb917d1b..8e7f6e2e817eb 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
@@ -20,10 +20,10 @@
 
 import org.apache.iotdb.lsm.context.RequestContext;
 import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
-import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.request.IRequest;
 
 /** basic lsm manager */
-public abstract class BasicLSMManager
+public abstract class BasicLSMManager
     implements ILSMManager {
 
   // the level process of the first layer of memory nodes
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
index 32974a7a5f0ee..f6cad902ab92d 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
@@ -19,10 +19,10 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.request.DeletionRequest;
+import org.apache.iotdb.lsm.request.IDeletionIRequest;
 
 /** manage deletion to MemTable */
-public class DeletionManager
+public class DeletionManager
     extends BasicLSMManager {
 
   // use wal manager object to write wal file on deletion
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
index 69547c48a3c5d..557a7998c99b8 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
@@ -19,12 +19,12 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.request.InsertionRequest;
+import org.apache.iotdb.lsm.request.IInsertionIRequest;
 
 import java.io.IOException;
 
 /** manage insertion to MemTable */
-public class InsertionManager
+public class InsertionManager
     extends BasicLSMManager {
 
   // use wal manager object to write wal file on insertion
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
index 3f828265d227a..392473953ff18 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
@@ -19,10 +19,10 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.request.QueryRequest;
+import org.apache.iotdb.lsm.request.IQueryIRequest;
 
 /** manage query to MemTable */
-public class QueryManager
+public class QueryManager
     extends BasicLSMManager {
   @Override
   public void preProcess(T root, R request, QueryRequestContext context) throws Exception {}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
index c6df6cebd56a6..47af7258e8a5e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.recover.IRecoverable;
-import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.request.IRequest;
 
 /** for memory structure recovery */
 public class RecoverManager {
@@ -38,7 +38,7 @@ public RecoverManager(WALManager walManager) {
    */
   public void recover(T t) throws Exception {
     while (true) {
-      Request request = walManager.read();
+      IRequest request = walManager.read();
       if (request == null) {
         walManager.setRecover(false);
         return;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
index 8d251a2189564..f742fadcfdf5f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.manager;
 
-import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.request.IRequest;
 import org.apache.iotdb.lsm.wal.IWALRecord;
 import org.apache.iotdb.lsm.wal.WALReader;
 import org.apache.iotdb.lsm.wal.WALWriter;
@@ -74,14 +74,14 @@ private void initFile(String schemaDirPath, String walFileName) throws IOExcepti
    * @param request request context
    * @throws IOException
    */
-  public abstract void write(Request request) throws IOException;
+  public abstract void write(IRequest request) throws IOException;
 
   /**
    * for recover
    *
    * @return request
    */
-  public abstract Request read();
+  public abstract IRequest read();
 
   public void close() throws IOException {
     walWriter.close();
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
index b0ea7441f43e2..f2fe7db7f860e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
@@ -18,9 +18,9 @@
  */
 package org.apache.iotdb.lsm.recover;
 
-import org.apache.iotdb.lsm.request.Request;
+import org.apache.iotdb.lsm.request.IRequest;
 
 public interface IRecoverable {
 
-   void recover(Request Request) throws Exception;
+   void recover(IRequest Request) throws Exception;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionIRequest.java
similarity index 80%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionIRequest.java
index bfc4c2f11d7a4..c6273a96c526b 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/InsertionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionIRequest.java
@@ -18,11 +18,14 @@
  */
 package org.apache.iotdb.lsm.request;
 
-public abstract class InsertionRequest extends Request {
+public interface IDeletionIRequest extends IRequest {
 
-  public InsertionRequest() {
-    requestType = RequestType.INSERT;
-  }
+  RequestType requestType = RequestType.DELETE;
+
+  V getValue();
 
-  public abstract V getValue();
+  @Override
+  default RequestType getRequestType() {
+    return requestType;
+  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionIRequest.java
similarity index 79%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionIRequest.java
index 3cdcb7ef145b6..3ae637503d40f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/QueryRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionIRequest.java
@@ -18,9 +18,14 @@
  */
 package org.apache.iotdb.lsm.request;
 
-public abstract class QueryRequest extends Request {
+public interface IInsertionIRequest extends IRequest {
 
-  public QueryRequest() {
-    requestType = RequestType.QUERY;
+  RequestType requestType = RequestType.INSERT;
+
+  V getValue();
+
+  @Override
+  default RequestType getRequestType() {
+    return requestType;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryIRequest.java
similarity index 81%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryIRequest.java
index 48dd02286b31b..2b5d14431808e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/DeletionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryIRequest.java
@@ -18,11 +18,12 @@
  */
 package org.apache.iotdb.lsm.request;
 
-public abstract class DeletionRequest extends Request {
+public interface IQueryIRequest extends IRequest {
 
-  public DeletionRequest() {
-    requestType = RequestType.DELETE;
-  }
+  RequestType requestType = RequestType.QUERY;
 
-  public abstract V getValue();
+  @Override
+  default RequestType getRequestType() {
+    return requestType;
+  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java
similarity index 66%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java
index 3ce5c516ba020..ba1eca7821bd2 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/Request.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java
@@ -22,27 +22,15 @@
 
 import java.util.List;
 
-public abstract class Request {
+public interface IRequest {
 
-  RequestType requestType;
+  K getKey(RequestContext context);
 
-  public Request() {
-    requestType = RequestType.NONE;
-  }
+  R getResult();
 
-  public abstract K getKey(RequestContext context);
+  void setResult(R result);
 
-  public abstract R getResult();
+  List getKeys();
 
-  public abstract void setResult(R result);
-
-  public abstract List getKeys();
-
-  public RequestType getRequestType() {
-    return requestType;
-  }
-
-  public void setRequestType(RequestType requestType) {
-    this.requestType = requestType;
-  }
+  RequestType getRequestType();
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java
index e88d3104eafb6..29429a074aab9 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java
@@ -22,6 +22,5 @@ public enum RequestType {
   NONE,
   INSERT,
   QUERY,
-  DELETE,
-  FLUSH;
+  DELETE
 }

From 168f0a6fe9059e5affa07d6c4594294d243a9d8d Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Mon, 24 Oct 2022 17:08:07 +0800
Subject: [PATCH 46/53] rename to ILevelProcessor

---
 .../tagIndex/deletion/MemChunkDeletion.java   |  8 +--
 .../deletion/MemChunkGroupDeletion.java       |  8 +--
 .../tagIndex/deletion/MemTableDeletion.java   |  9 +--
 .../deletion/MemTableGroupDeletion.java       |  8 +--
 .../insertion/MemChunkGroupInsertion.java     |  8 +--
 .../tagIndex/insertion/MemChunkInsertion.java |  8 +--
 .../insertion/MemTableGroupInsertion.java     |  8 +--
 .../tagIndex/insertion/MemTableInsertion.java |  8 +--
 .../tagIndex/query/MemChunkGroupQuery.java    |  8 +--
 .../tagIndex/query/MemChunkQuery.java         |  8 +--
 .../tagIndex/query/MemTableGroupQuery.java    |  8 +--
 .../tagIndex/query/MemTableQuery.java         |  8 +--
 ...ionProcess.java => DeletionProcessor.java} |  2 +-
 ...onProcess.java => InsertionProcessor.java} |  2 +-
 ...{QueryProcess.java => QueryProcessor.java} |  4 +-
 .../iotdb/lsm/engine/LSMEngineDirector.java   | 10 +--
 ...lProcess.java => BasicLevelProcessor.java} | 10 +--
 ...Process.java => DeleteLevelProcessor.java} |  4 +-
 ...lProcess.java => FlushLevelProcessor.java} |  4 +-
 ...LevelProcess.java => ILevelProcessor.java} |  4 +-
 ...Process.java => InsertLevelProcessor.java} |  4 +-
 .../lsm/levelProcess/LevelProcessChain.java   |  4 +-
 ...lProcess.java => QueryLevelProcessor.java} |  4 +-
 .../lsm/property/PropertyDescriptor.java      | 71 ++++++++-----------
 .../iotdb/lsm/strategy/BFSAccessStrategy.java |  4 +-
 .../iotdb/lsm/strategy/IAccessStrategy.java   |  4 +-
 .../lsm/strategy/PostOrderAccessStrategy.java |  4 +-
 .../lsm/strategy/PreOrderAccessStrategy.java  |  4 +-
 .../lsm/strategy/RBFSAccessStrategy.java      |  4 +-
 29 files changed, 114 insertions(+), 126 deletions(-)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/{DeletionProcess.java => DeletionProcessor.java} (96%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/{InsertionProcess.java => InsertionProcessor.java} (96%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/{QueryProcess.java => QueryProcessor.java} (94%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/{BasicLevelProcess.java => BasicLevelProcessor.java} (87%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/{DeleteLevelProcess.java => DeleteLevelProcessor.java} (91%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/{FlushLevelProcess.java => FlushLevelProcessor.java} (91%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/{ILevelProcess.java => ILevelProcessor.java} (90%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/{InsertLevelProcess.java => InsertLevelProcessor.java} (91%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/{QueryLevelProcess.java => QueryLevelProcessor.java} (91%)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
index 52c5e69e52404..6520ec8bb74a3 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
@@ -20,15 +20,15 @@
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
-import org.apache.iotdb.lsm.annotation.DeletionProcess;
+import org.apache.iotdb.lsm.annotation.DeletionProcessor;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor;
 
 import java.util.List;
 
 /** deletion for MemChunk */
-@DeletionProcess(level = 3)
-public class MemChunkDeletion extends DeleteLevelProcess {
+@DeletionProcessor(level = 3)
+public class MemChunkDeletion extends DeleteLevelProcessor {
 
   /**
    * MemChunk is the last layer of memory nodes, no children
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
index 42451293a2e81..c00dafa140843 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
@@ -21,17 +21,17 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
-import org.apache.iotdb.lsm.annotation.DeletionProcess;
+import org.apache.iotdb.lsm.annotation.DeletionProcessor;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 
 /** deletion for MemChunkGroup */
-@DeletionProcess(level = 2)
+@DeletionProcessor(level = 2)
 public class MemChunkGroupDeletion
-    extends DeleteLevelProcess {
+    extends DeleteLevelProcessor {
 
   /**
    * get all MemChunks that need to be processed in the current MemChunkGroup
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
index 91f9d45c03d2d..25ec70c5cf93a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
@@ -21,17 +21,18 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
-import org.apache.iotdb.lsm.annotation.DeletionProcess;
+import org.apache.iotdb.lsm.annotation.DeletionProcessor;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
 /** deletion for MemTable */
-@DeletionProcess(level = 1)
-public class MemTableDeletion extends DeleteLevelProcess {
+@DeletionProcessor(level = 1)
+public class MemTableDeletion
+    extends DeleteLevelProcessor {
 
   /**
    * get all MemChunkGroups that need to be processed in the current MemTable
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
index 7094681a5baf5..bb9d1573c455f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
@@ -21,16 +21,16 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
-import org.apache.iotdb.lsm.annotation.DeletionProcess;
+import org.apache.iotdb.lsm.annotation.DeletionProcessor;
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 
-@DeletionProcess(level = 0)
+@DeletionProcessor(level = 0)
 public class MemTableGroupDeletion
-    extends DeleteLevelProcess {
+    extends DeleteLevelProcessor {
   @Override
   public List getChildren(
       MemTableGroup memNode, DeletionRequest request, DeleteRequestContext context) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
index 66bb854ec12e4..07a58b4483ebb 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
@@ -21,17 +21,17 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
-import org.apache.iotdb.lsm.annotation.InsertionProcess;
+import org.apache.iotdb.lsm.annotation.InsertionProcessor;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 
 /** insertion for MemChunkGroup */
-@InsertionProcess(level = 2)
+@InsertionProcessor(level = 2)
 public class MemChunkGroupInsertion
-    extends InsertLevelProcess {
+    extends InsertLevelProcessor {
 
   /**
    * get all MemChunks that need to be processed in the current MemChunkGroup
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
index e5df48d19fe97..c80793dc603f7 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
@@ -20,15 +20,15 @@
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
-import org.apache.iotdb.lsm.annotation.InsertionProcess;
+import org.apache.iotdb.lsm.annotation.InsertionProcessor;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor;
 
 import java.util.List;
 
 /** insertion for MemChunk */
-@InsertionProcess(level = 3)
-public class MemChunkInsertion extends InsertLevelProcess {
+@InsertionProcessor(level = 3)
+public class MemChunkInsertion extends InsertLevelProcessor {
 
   /**
    * MemChunk is the last layer of memory nodes, no children
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
index b5d57aae7aed1..bef19b9532e6b 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
@@ -22,17 +22,17 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
-import org.apache.iotdb.lsm.annotation.InsertionProcess;
+import org.apache.iotdb.lsm.annotation.InsertionProcessor;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-@InsertionProcess(level = 0)
+@InsertionProcessor(level = 0)
 public class MemTableGroupInsertion
-    extends InsertLevelProcess {
+    extends InsertLevelProcessor {
 
   @Override
   public List getChildren(
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
index f0faf7bad1e71..16b4c7f4cc44a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
@@ -21,17 +21,17 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
-import org.apache.iotdb.lsm.annotation.InsertionProcess;
+import org.apache.iotdb.lsm.annotation.InsertionProcessor;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 
 /** insertion for MemTable */
-@InsertionProcess(level = 1)
+@InsertionProcessor(level = 1)
 public class MemTableInsertion
-    extends InsertLevelProcess {
+    extends InsertLevelProcessor {
 
   /**
    * get all MemChunkGroups that need to be processed in the current MemTable
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
index 7d355db1f9d3f..a6001f425f808 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
@@ -21,16 +21,16 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
-import org.apache.iotdb.lsm.annotation.QueryProcess;
+import org.apache.iotdb.lsm.annotation.QueryProcessor;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 
 /** query for MemChunkGroup */
-@QueryProcess(level = 2)
-public class MemChunkGroupQuery extends QueryLevelProcess {
+@QueryProcessor(level = 2)
+public class MemChunkGroupQuery extends QueryLevelProcessor {
 
   /**
    * get all MemChunks that need to be processed in the current MemChunkGroup
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
index aaea71f5eb1c5..96d389f7520df 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
@@ -20,17 +20,17 @@
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
-import org.apache.iotdb.lsm.annotation.QueryProcess;
+import org.apache.iotdb.lsm.annotation.QueryProcessor;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor;
 
 import org.roaringbitmap.RoaringBitmap;
 
 import java.util.List;
 
 /** query for MemChunk */
-@QueryProcess(level = 3)
-public class MemChunkQuery extends QueryLevelProcess {
+@QueryProcessor(level = 3)
+public class MemChunkQuery extends QueryLevelProcessor {
 
   /**
    * MemChunk is the last layer of memory nodes, no children
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
index f24c7decc19cf..96af3f18da691 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
@@ -21,15 +21,15 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
-import org.apache.iotdb.lsm.annotation.QueryProcess;
+import org.apache.iotdb.lsm.annotation.QueryProcessor;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 
-@QueryProcess(level = 0)
-public class MemTableGroupQuery extends QueryLevelProcess {
+@QueryProcessor(level = 0)
+public class MemTableGroupQuery extends QueryLevelProcessor {
   @Override
   public List getChildren(
       MemTableGroup memNode, QueryRequest request, QueryRequestContext context) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
index d27dfba3347bf..ff5d6c00c2e3f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
@@ -21,9 +21,9 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
-import org.apache.iotdb.lsm.annotation.QueryProcess;
+import org.apache.iotdb.lsm.annotation.QueryProcessor;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor;
 
 import org.roaringbitmap.RoaringBitmap;
 
@@ -32,8 +32,8 @@
 import java.util.Set;
 
 /** query for MemTable */
-@QueryProcess(level = 1)
-public class MemTableQuery extends QueryLevelProcess {
+@QueryProcessor(level = 1)
+public class MemTableQuery extends QueryLevelProcessor {
 
   /**
    * get all MemChunkGroups that need to be processed in the current MemTable
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java
similarity index 96%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java
index 847b42f421cd5..82220f23163c0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java
@@ -25,7 +25,7 @@
 
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
-public @interface DeletionProcess {
+public @interface DeletionProcessor {
   // level of the levelProcess
   int level() default -1;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java
similarity index 96%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java
index 9af1c450450ff..7d3c24aa70e66 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java
@@ -25,7 +25,7 @@
 
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
-public @interface InsertionProcess {
+public @interface InsertionProcessor {
   // level of the levelProcess
   int level() default -1;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java
similarity index 94%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java
index 4a712c082130e..abebbfc241fb1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java
@@ -25,7 +25,7 @@
 
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
-public @interface QueryProcess {
-  // level of the levelProcess
+public @interface QueryProcessor {
+  // level of the levelProcessor
   int level() default -1;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
index 8b0bb76f2265a..4dc8e961f796e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
@@ -23,7 +23,7 @@
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.ILevelProcess;
+import org.apache.iotdb.lsm.levelProcess.ILevelProcessor;
 import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
 import org.apache.iotdb.lsm.manager.WALManager;
 import org.apache.iotdb.lsm.property.Property;
@@ -89,7 +89,7 @@ LevelProcessChain generateLevelProcessChain(List levelProcessCl
     LevelProcessChain levelProcessChain = new LevelProcessChain<>();
     try {
       if (levelProcessClassNames.size() > 0) {
-        ILevelProcess iLevelProcess =
+        ILevelProcessor iLevelProcess =
             levelProcessChain.nextLevel(generateLevelProcess(levelProcessClassNames.get(0)));
         for (int i = 1; i < levelProcessClassNames.size(); i++) {
           iLevelProcess =
@@ -103,12 +103,12 @@ LevelProcessChain generateLevelProcessChain(List levelProcessCl
   }
 
   private 
-      ILevelProcess generateLevelProcess(String className)
+      ILevelProcessor generateLevelProcess(String className)
           throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException,
               InstantiationException, IllegalAccessException {
     Class c = Class.forName(className);
-    ILevelProcess result =
-        (ILevelProcess) c.getDeclaredConstructor().newInstance();
+    ILevelProcessor result =
+        (ILevelProcessor) c.getDeclaredConstructor().newInstance();
     return result;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java
similarity index 87%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java
index 0173f3ab3866e..0b594aaedaea4 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java
@@ -23,11 +23,11 @@
 import java.util.List;
 
 /** the processing method corresponding to each layer of memory nodes */
-public abstract class BasicLevelProcess
-    implements ILevelProcess {
+public abstract class BasicLevelProcessor
+    implements ILevelProcessor {
 
   // the next level process
-  ILevelProcess next;
+  ILevelProcessor next;
 
   /**
    * process the current layer memory node
@@ -53,7 +53,7 @@ public abstract class BasicLevelProcess
    * @return the next level process
    */
   @Override
-  public  ILevelProcess nextLevel(ILevelProcess next) {
+  public  ILevelProcessor nextLevel(ILevelProcessor next) {
     this.next = next;
     return next;
   }
@@ -73,7 +73,7 @@ public boolean hasNext() {
     return next != null;
   }
 
-  public ILevelProcess getNext() {
+  public ILevelProcessor getNext() {
     return next;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java
similarity index 91%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java
index 850c705c77149..2e511b144d3b0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java
@@ -21,8 +21,8 @@
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 
 /** indicates the deletion method of each layer of memory nodes */
-public abstract class DeleteLevelProcess
-    extends BasicLevelProcess {
+public abstract class DeleteLevelProcessor
+    extends BasicLevelProcessor {
 
   /**
    * the deletion method of memory node
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java
similarity index 91%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java
index 8da40d163efbc..0b14ce8e9dce4 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java
@@ -21,8 +21,8 @@
 import org.apache.iotdb.lsm.context.FlushRequestContext;
 
 /** indicates the flush method of each layer of memory nodes */
-public abstract class FlushLevelProcess
-    extends BasicLevelProcess {
+public abstract class FlushLevelProcessor
+    extends BasicLevelProcessor {
 
   /**
    * the flush method of memory node
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java
similarity index 90%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java
index 38368d32c3102..ddf45b2489e2a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java
@@ -21,7 +21,7 @@
 import org.apache.iotdb.lsm.context.RequestContext;
 
 /** the processing method corresponding to each layer of memory nodes */
-public interface ILevelProcess {
+public interface ILevelProcessor {
 
   /**
    * add the LevelProcess of the next layer of memory nodes
@@ -29,7 +29,7 @@ public interface ILevelProcess {
    * @param next LevelProcess of the next layer
    * @return LevelProcess of the next layer
    */
-   ILevelProcess nextLevel(ILevelProcess next);
+   ILevelProcessor nextLevel(ILevelProcessor next);
 
   /**
    * use this method to process memory nodes at each layer according to the access strategy
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java
similarity index 91%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java
index ed6a7af6bb63d..c231e8f635d39 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java
@@ -21,8 +21,8 @@
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 
 /** indicates the insertion method of each layer of memory nodes */
-public abstract class InsertLevelProcess
-    extends BasicLevelProcess {
+public abstract class InsertLevelProcessor
+    extends BasicLevelProcessor {
 
   /**
    * the insertion method of memory node
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
index 9f89bda3377b2..942fd82abac89 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
@@ -24,9 +24,9 @@
 public class LevelProcessChain {
 
   // the level process of the first layer of memory nodes
-  ILevelProcess headLevelProcess;
+  ILevelProcessor headLevelProcess;
 
-  public  ILevelProcess nextLevel(ILevelProcess next) {
+  public  ILevelProcessor nextLevel(ILevelProcessor next) {
     this.headLevelProcess = next;
     return next;
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java
similarity index 91%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java
index 93f1c481f4d1b..c7700f41193b1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java
@@ -21,8 +21,8 @@
 import org.apache.iotdb.lsm.context.QueryRequestContext;
 
 /** indicates the query method of each layer of memory nodes */
-public abstract class QueryLevelProcess
-    extends BasicLevelProcess {
+public abstract class QueryLevelProcessor
+    extends BasicLevelProcessor {
 
   /**
    * the query method of memory node
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
index c245a1dfa40cf..ac4ec72c8e079 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
@@ -18,14 +18,15 @@
  */
 package org.apache.iotdb.lsm.property;
 
-import org.apache.iotdb.lsm.annotation.DeletionProcess;
-import org.apache.iotdb.lsm.annotation.InsertionProcess;
-import org.apache.iotdb.lsm.annotation.QueryProcess;
+import org.apache.iotdb.lsm.annotation.DeletionProcessor;
+import org.apache.iotdb.lsm.annotation.InsertionProcessor;
+import org.apache.iotdb.lsm.annotation.QueryProcessor;
 
 import org.reflections.Reflections;
 import org.reflections.util.ConfigurationBuilder;
 import org.reflections.util.FilterBuilder;
 
+import java.lang.annotation.Annotation;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -45,60 +46,46 @@ public static Property getProperty(String packageName) throws Exception {
     return property;
   }
 
-  private static void setInsertionLevelProcess(Property property, Reflections reflections)
-      throws Exception {
-    Set> annotated = reflections.getTypesAnnotatedWith(InsertionProcess.class);
+  private static void setInsertionLevelProcess(Property property, Reflections reflections) {
+    Set> annotated = reflections.getTypesAnnotatedWith(InsertionProcessor.class);
     List levelProcessClass = new ArrayList<>();
     for (Class clz : annotated) {
-      InsertionProcess annotationInfo = clz.getAnnotation(InsertionProcess.class);
-      int level = annotationInfo.level();
-      if (level < levelProcessClass.size()) {
-        levelProcessClass.set(level, clz.getName());
-      } else {
-        for (int i = levelProcessClass.size(); i < level; i++) {
-          levelProcessClass.add("");
-        }
-        levelProcessClass.add(clz.getName());
-      }
+      InsertionProcessor annotationInfo = clz.getAnnotation(InsertionProcessor.class);
+      setLevelProcessors(levelProcessClass, clz, annotationInfo.level());
     }
     property.setInsertionLevelProcessClass(levelProcessClass);
   }
 
-  private static void setDeletionLevelProcess(Property property, Reflections reflections)
-      throws Exception {
-    Set> annotated = reflections.getTypesAnnotatedWith(DeletionProcess.class);
+  private static void setDeletionLevelProcess(Property property, Reflections reflections) {
+    Set> annotated = reflections.getTypesAnnotatedWith(DeletionProcessor.class);
     List levelProcessClass = new ArrayList<>();
     for (Class clz : annotated) {
-      DeletionProcess annotationInfo = clz.getAnnotation(DeletionProcess.class);
-      int level = annotationInfo.level();
-      if (level < levelProcessClass.size()) {
-        levelProcessClass.set(level, clz.getName());
-      } else {
-        for (int i = levelProcessClass.size(); i < level; i++) {
-          levelProcessClass.add("");
-        }
-        levelProcessClass.add(clz.getName());
-      }
+      DeletionProcessor annotationInfo = clz.getAnnotation(DeletionProcessor.class);
+      setLevelProcessors(levelProcessClass, clz, annotationInfo.level());
     }
     property.setDeletionLevelProcessClass(levelProcessClass);
   }
 
-  private static void setQueryLevelProcess(Property property, Reflections reflections)
-      throws Exception {
+  private static  void setQueryLevelProcess(
+      Property property, Reflections reflections) {
     List levelProcessClass = new ArrayList<>();
-    Set> annotated = reflections.getTypesAnnotatedWith(QueryProcess.class);
+    Set> annotated = reflections.getTypesAnnotatedWith(QueryProcessor.class);
     for (Class clz : annotated) {
-      QueryProcess annotationInfo = clz.getAnnotation(QueryProcess.class);
-      int level = annotationInfo.level();
-      if (level < levelProcessClass.size()) {
-        levelProcessClass.set(level, clz.getName());
-      } else {
-        for (int i = levelProcessClass.size(); i < level; i++) {
-          levelProcessClass.add("");
-        }
-        levelProcessClass.add(clz.getName());
-      }
+      QueryProcessor annotationInfo = clz.getAnnotation(QueryProcessor.class);
+      setLevelProcessors(levelProcessClass, clz, annotationInfo.level());
     }
     property.setQueryLevelProcessClass(levelProcessClass);
   }
+
+  private static  void setLevelProcessors(
+      List levelProcessClass, Class clz, int level) {
+    if (level < levelProcessClass.size()) {
+      levelProcessClass.set(level, clz.getName());
+    } else {
+      for (int i = levelProcessClass.size(); i < level; i++) {
+        levelProcessClass.add("");
+      }
+      levelProcessClass.add(clz.getName());
+    }
+  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
index 1bb929e53d2eb..e9870d410bac0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.strategy;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.LinkedList;
@@ -41,7 +41,7 @@ public class BFSAccessStrategy implements IAccessStrategy {
    */
   @Override
   public  void execute(
-      BasicLevelProcess levelProcess, I memNode, R request, C context) {
+      BasicLevelProcessor levelProcess, I memNode, R request, C context) {
     List children = new ArrayList<>();
     int currentLevel = context.getLevel();
     if (sameLevelMemNodes == null) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java
index e78a7b2ca00ab..89dc1d3cd9b63 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.strategy;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 /** access strategy for memory nodes */
 public interface IAccessStrategy {
@@ -32,5 +32,5 @@ public interface IAccessStrategy {
    * @param context request context
    */
    void execute(
-      BasicLevelProcess levelProcess, I memNode, R request, C context);
+      BasicLevelProcessor levelProcess, I memNode, R request, C context);
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
index b04889ca02be0..46cc762092d65 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.strategy;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 import java.util.List;
 
@@ -35,7 +35,7 @@ public class PostOrderAccessStrategy implements IAccessStrategy {
    */
   @Override
   public  void execute(
-      BasicLevelProcess levelProcess, I memNode, R request, C context) {
+      BasicLevelProcessor levelProcess, I memNode, R request, C context) {
     int currentLevel = context.getLevel();
     IAccessStrategy accessStrategy = context.getAccessStrategy();
     // get all memory nodes to be processed in the next layer
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
index c6b5ba76dbda1..0ed5057671207 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.strategy;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 import java.util.List;
 
@@ -35,7 +35,7 @@ public class PreOrderAccessStrategy implements IAccessStrategy {
    */
   @Override
   public  void execute(
-      BasicLevelProcess levelProcess, I memNode, R request, C context) {
+      BasicLevelProcessor levelProcess, I memNode, R request, C context) {
     int currentLevel = context.getLevel();
     // process the current memory node
     levelProcess.handle(memNode, request, context);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
index f89861faa78a1..2b2f2a5744620 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.strategy;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess;
+import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 import java.util.List;
 
@@ -35,7 +35,7 @@ public class RBFSAccessStrategy implements IAccessStrategy {
    */
   @Override
   public  void execute(
-      BasicLevelProcess levelProcess, I memNode, R request, C context) {
+      BasicLevelProcessor levelProcess, I memNode, R request, C context) {
     int currentLevel = context.getLevel();
 
     // if the upper bound has not been set and there is no next-level processing method, set the

From 00a8240d97fec9639351e0bcc781f509e4fff866 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Mon, 24 Oct 2022 17:26:01 +0800
Subject: [PATCH 47/53] remove Exception

---
 .../tagIndex/wal/WALManager.java              | 30 ++++++++++++-------
 .../apache/iotdb/lsm/engine/ILSMEngine.java   | 24 ++++-----------
 .../apache/iotdb/lsm/engine/LSMEngine.java    | 30 ++++---------------
 .../iotdb/lsm/manager/BasicLSMManager.java    |  2 +-
 .../iotdb/lsm/manager/DeletionManager.java    |  4 +--
 .../apache/iotdb/lsm/manager/ILSMManager.java |  6 ++--
 .../iotdb/lsm/manager/InsertionManager.java   |  7 ++---
 .../iotdb/lsm/manager/QueryManager.java       |  4 +--
 .../iotdb/lsm/manager/RecoverManager.java     |  2 +-
 .../apache/iotdb/lsm/manager/WALManager.java  |  2 +-
 .../lsm/property/PropertyDescriptor.java      |  3 +-
 .../iotdb/lsm/recover/IRecoverable.java       |  2 +-
 12 files changed, 45 insertions(+), 71 deletions(-)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
index 59549b2d07fb9..f8c9a1eaec612 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java
@@ -24,10 +24,16 @@
 import org.apache.iotdb.lsm.wal.IWALRecord;
 import org.apache.iotdb.lsm.wal.WALReader;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.IOException;
 
 /** Manage wal entry writes and reads */
 public class WALManager extends org.apache.iotdb.lsm.manager.WALManager {
+
+  private static final Logger logger = LoggerFactory.getLogger(WALManager.class);
+
   private static final int INSERT = 1;
 
   private static final int DELETE = 2;
@@ -53,17 +59,21 @@ public WALManager(String schemaDirPath) {
    * @throws IOException
    */
   @Override
-  public synchronized void write(IRequest request) throws IOException {
+  public synchronized void write(IRequest request) {
     if (isRecover()) return;
-    switch (request.getRequestType()) {
-      case INSERT:
-        process((InsertionRequest) request);
-        break;
-      case DELETE:
-        process((DeletionRequest) request);
-        break;
-      default:
-        break;
+    try {
+      switch (request.getRequestType()) {
+        case INSERT:
+          process((InsertionRequest) request);
+          break;
+        case DELETE:
+          process((DeletionRequest) request);
+          break;
+        default:
+          break;
+      }
+    } catch (IOException e) {
+      logger.error(e.getMessage());
     }
   }
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
index 33860c644f243..75c472de57f55 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
@@ -18,9 +18,7 @@
  */
 package org.apache.iotdb.lsm.engine;
 
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.commons.utils.TestOnly;
 import org.apache.iotdb.lsm.request.IDeletionIRequest;
 import org.apache.iotdb.lsm.request.IInsertionIRequest;
 import org.apache.iotdb.lsm.request.IQueryIRequest;
@@ -29,24 +27,14 @@
 
 public interface ILSMEngine {
 
-   void insert(IInsertionIRequest insertionRequest) throws Exception;
+   void insert(IInsertionIRequest insertionRequest);
 
-   void insert(
-      IInsertionIRequest insertionRequest, InsertRequestContext insertRequestContext)
-      throws Exception;
+   void query(IQueryIRequest queryRequest);
 
-   void query(IQueryIRequest queryRequest) throws Exception;
+   void delete(IDeletionIRequest deletionRequest);
 
-   void query(IQueryIRequest queryRequest, QueryRequestContext queryRequestContext)
-      throws Exception;
-
-   void delete(IDeletionIRequest deletionRequest) throws Exception;
-
-   void delete(
-      IDeletionIRequest deletionRequest, DeleteRequestContext deleteRequestContext)
-      throws Exception;
-
-  void recover() throws Exception;
+  void recover();
 
+  @TestOnly
   void clear() throws IOException;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
index de761df03166a..dd30c5b5caa64 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
@@ -55,42 +55,22 @@ public void setRootMemNode(T rootMemNode) {
   }
 
   @Override
-  public  void insert(IInsertionIRequest insertionRequest) throws Exception {
+  public  void insert(IInsertionIRequest insertionRequest) {
     insertionManager.process(rootMemNode, insertionRequest, new InsertRequestContext());
   }
 
   @Override
-  public  void insert(
-      IInsertionIRequest insertionRequest, InsertRequestContext insertRequestContext)
-      throws Exception {
-    insertionManager.process(rootMemNode, insertionRequest, insertRequestContext);
-  }
-
-  @Override
-  public  void query(IQueryIRequest queryRequest) throws Exception {
+  public  void query(IQueryIRequest queryRequest) {
     queryManager.process(rootMemNode, queryRequest, new QueryRequestContext());
   }
 
   @Override
-  public  void query(
-      IQueryIRequest queryRequest, QueryRequestContext queryRequestContext) throws Exception {
-    queryManager.process(rootMemNode, queryRequest, queryRequestContext);
-  }
-
-  @Override
-  public  void delete(IDeletionIRequest deletionRequest) throws Exception {
+  public  void delete(IDeletionIRequest deletionRequest) {
     deletionManager.process(rootMemNode, deletionRequest, new DeleteRequestContext());
   }
 
   @Override
-  public  void delete(
-      IDeletionIRequest deletionRequest, DeleteRequestContext deleteRequestContext)
-      throws Exception {
-    deletionManager.process(rootMemNode, deletionRequest, deleteRequestContext);
-  }
-
-  @Override
-  public void recover() throws Exception {
+  public void recover() {
     recoverManager.recover(this);
   }
 
@@ -126,7 +106,7 @@ protected void setRecoverManager(RecoverManager> recoverManager) {
   }
 
   @Override
-  public  void recover(IRequest request) throws Exception {
+  public  void recover(IRequest request) {
     switch (request.getRequestType()) {
       case INSERT:
         insert((IInsertionIRequest) request);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
index 8e7f6e2e817eb..13f55dd425752 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
@@ -43,7 +43,7 @@ public BasicLSMManager(LevelProcessChain levelProcessChain) {
    * @throws Exception
    */
   @Override
-  public void process(T root, R request, C context) throws Exception {
+  public void process(T root, R request, C context) {
     preProcess(root, request, context);
     levelProcessChain.process(root, request, context);
     postProcess(root, request, context);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
index f6cad902ab92d..17286d3b968b1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
@@ -40,10 +40,10 @@ public DeletionManager(WALManager walManager) {
    * @throws Exception
    */
   @Override
-  public void preProcess(T root, R deletionRequest, DeleteRequestContext context) throws Exception {
+  public void preProcess(T root, R deletionRequest, DeleteRequestContext context) {
     walManager.write(deletionRequest);
   }
 
   @Override
-  public void postProcess(T root, R request, DeleteRequestContext context) throws Exception {}
+  public void postProcess(T root, R request, DeleteRequestContext context) {}
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java
index 1d6afa9879865..0bffc32acbc5e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java
@@ -30,7 +30,7 @@ public interface ILSMManager {
    * @param context request context
    * @throws Exception
    */
-  void preProcess(T root, R request, C context) throws Exception;
+  void preProcess(T root, R request, C context);
 
   /**
    * postprocessing of the root memory node
@@ -39,7 +39,7 @@ public interface ILSMManager {
    * @param context request context
    * @throws Exception
    */
-  void postProcess(T root, R request, C context) throws Exception;
+  void postProcess(T root, R request, C context);
 
   /**
    * use this method to process root memory node
@@ -47,5 +47,5 @@ public interface ILSMManager {
    * @param memNode memory node
    * @param context request context
    */
-  void process(T memNode, R request, C context) throws Exception;
+  void process(T memNode, R request, C context);
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
index 557a7998c99b8..3855ea1bc19b0 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
@@ -21,8 +21,6 @@
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.request.IInsertionIRequest;
 
-import java.io.IOException;
-
 /** manage insertion to MemTable */
 public class InsertionManager
     extends BasicLSMManager {
@@ -42,11 +40,10 @@ public InsertionManager(WALManager walManager) {
    * @throws Exception
    */
   @Override
-  public void preProcess(T root, R insertionRequest, InsertRequestContext context)
-      throws IOException {
+  public void preProcess(T root, R insertionRequest, InsertRequestContext context) {
     walManager.write(insertionRequest);
   }
 
   @Override
-  public void postProcess(T root, R request, InsertRequestContext context) throws Exception {}
+  public void postProcess(T root, R request, InsertRequestContext context) {}
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
index 392473953ff18..fd1055ed0521d 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
@@ -25,8 +25,8 @@
 public class QueryManager
     extends BasicLSMManager {
   @Override
-  public void preProcess(T root, R request, QueryRequestContext context) throws Exception {}
+  public void preProcess(T root, R request, QueryRequestContext context) {}
 
   @Override
-  public void postProcess(T root, R request, QueryRequestContext context) throws Exception {}
+  public void postProcess(T root, R request, QueryRequestContext context) {}
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
index 47af7258e8a5e..f3aafee5720e7 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
@@ -36,7 +36,7 @@ public RecoverManager(WALManager walManager) {
    *
    * @param t
    */
-  public void recover(T t) throws Exception {
+  public void recover(T t) {
     while (true) {
       IRequest request = walManager.read();
       if (request == null) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
index f742fadcfdf5f..8d280ab0a498a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
@@ -74,7 +74,7 @@ private void initFile(String schemaDirPath, String walFileName) throws IOExcepti
    * @param request request context
    * @throws IOException
    */
-  public abstract void write(IRequest request) throws IOException;
+  public abstract void write(IRequest request);
 
   /**
    * for recover
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
index ac4ec72c8e079..e40ac7af97e23 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
@@ -77,8 +77,7 @@ private static  void setQueryLevelProcess(
     property.setQueryLevelProcessClass(levelProcessClass);
   }
 
-  private static  void setLevelProcessors(
-      List levelProcessClass, Class clz, int level) {
+  private static void setLevelProcessors(List levelProcessClass, Class clz, int level) {
     if (level < levelProcessClass.size()) {
       levelProcessClass.set(level, clz.getName());
     } else {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
index f2fe7db7f860e..e82905b795f39 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
@@ -22,5 +22,5 @@
 
 public interface IRecoverable {
 
-   void recover(IRequest Request) throws Exception;
+   void recover(IRequest Request);
 }

From a5b7f336e8e788fc86176df8a029fefe69182ef2 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Mon, 24 Oct 2022 22:07:01 +0800
Subject: [PATCH 48/53] delete LSMEngineDirector.java

---
 .../tagIndex/Request/DeletionRequest.java     |   4 +-
 .../tagIndex/Request/InsertionRequest.java    |   4 +-
 .../tagIndex/Request/QueryRequest.java        |   4 +-
 .../tagIndex/TagInvertedIndex.java            |  13 +-
 .../lsm/annotation/DeletionProcessor.java     |   7 +-
 .../lsm/annotation/InsertionProcessor.java    |   7 +-
 .../iotdb/lsm/annotation/QueryProcessor.java  |   7 +-
 .../apache/iotdb/lsm/engine/ILSMEngine.java   |  21 +++-
 .../apache/iotdb/lsm/engine/LSMEngine.java    |  34 +++---
 .../iotdb/lsm/engine/LSMEngineBuilder.java    | 108 +++++++++++++++--
 .../iotdb/lsm/engine/LSMEngineDirector.java   | 114 ------------------
 ...essChain.java => LevelProcessorChain.java} |   2 +-
 .../iotdb/lsm/manager/BasicLSMManager.java    |   8 +-
 .../iotdb/lsm/manager/DeletionManager.java    |   4 +-
 .../iotdb/lsm/manager/InsertionManager.java   |   4 +-
 .../iotdb/lsm/manager/QueryManager.java       |   4 +-
 ...Descriptor.java => PropertyGenerator.java} |   4 +-
 ...ionIRequest.java => IDeletionRequest.java} |   2 +-
 ...onIRequest.java => IInsertionRequest.java} |   2 +-
 ...IQueryIRequest.java => IQueryRequest.java} |   2 +-
 20 files changed, 175 insertions(+), 180 deletions(-)
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/{LevelProcessChain.java => LevelProcessorChain.java} (94%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/{PropertyDescriptor.java => PropertyGenerator.java} (97%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/{IDeletionIRequest.java => IDeletionRequest.java} (93%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/{IInsertionIRequest.java => IInsertionRequest.java} (93%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/{IQueryIRequest.java => IQueryRequest.java} (93%)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
index 601220ad64a0a..6d2e1b21f6ab1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
@@ -19,11 +19,11 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.request.IDeletionIRequest;
+import org.apache.iotdb.lsm.request.IDeletionRequest;
 
 import java.util.List;
 
-public class DeletionRequest implements IDeletionIRequest {
+public class DeletionRequest implements IDeletionRequest {
 
   List keys;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
index 716fb9909c0e0..b97c66b2dbd70 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
@@ -19,11 +19,11 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.request.IInsertionIRequest;
+import org.apache.iotdb.lsm.request.IInsertionRequest;
 
 import java.util.List;
 
-public class InsertionRequest implements IInsertionIRequest {
+public class InsertionRequest implements IInsertionRequest {
 
   List keys;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
index acb48036db03c..7cd72d749e289 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
@@ -19,13 +19,13 @@
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.request.IQueryIRequest;
+import org.apache.iotdb.lsm.request.IQueryRequest;
 
 import org.roaringbitmap.RoaringBitmap;
 
 import java.util.List;
 
-public class QueryRequest implements IQueryIRequest {
+public class QueryRequest implements IQueryRequest {
 
   List keys;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index 49f062f9fe40e..8b641e09265b5 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -28,7 +28,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
 import org.apache.iotdb.lsm.engine.LSMEngine;
-import org.apache.iotdb.lsm.engine.LSMEngineDirector;
+import org.apache.iotdb.lsm.engine.LSMEngineBuilder;
 
 import org.roaringbitmap.RoaringBitmap;
 import org.slf4j.Logger;
@@ -61,11 +61,14 @@ public TagInvertedIndex(String schemaDirPath) {
               tagSchemaConfig.getWalBufferSize(),
               new WALEntry(),
               false);
-      LSMEngineDirector lsmEngineDirector = new LSMEngineDirector<>();
+      MemTableGroup memTableGroup =
+          new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable());
+      LSMEngineBuilder lsmEngineBuilder = new LSMEngineBuilder<>();
       lsmEngine =
-          lsmEngineDirector.getLSMEngine(
-              "org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex", walManager);
-      lsmEngine.setRootMemNode(new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable()));
+          lsmEngineBuilder
+              .buildLSMManagers("org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex", walManager)
+              .buildRootMemNode(memTableGroup)
+              .build();
       lsmEngine.recover();
     } catch (Exception e) {
       logger.error(e.getMessage());
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java
index 82220f23163c0..e4d1d6e8b228f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java
@@ -23,9 +23,14 @@
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
+/**
+ * Indicates that the class using this annotation is a DeleteLevelProcessor, and the attribute level
+ * of the annotation indicates which layer of memory node the deletion method of the
+ * DeleteLevelProcessor is.
+ */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
 public @interface DeletionProcessor {
-  // level of the levelProcess
+  // level of the DeleteLevelProcessor
   int level() default -1;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java
index 7d3c24aa70e66..c06a085eed810 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java
@@ -23,9 +23,14 @@
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
+/**
+ * Indicates that the class using this annotation is a InsertLevelProcessor, and the attribute level
+ * of the annotation indicates which layer of memory node the insertion method of the
+ * InsertLevelProcessor is.
+ */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
 public @interface InsertionProcessor {
-  // level of the levelProcess
+  // level of the InsertLevelProcessor
   int level() default -1;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java
index abebbfc241fb1..f7734b68989a5 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java
@@ -23,9 +23,14 @@
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
+/**
+ * Indicates that the class using this annotation is a QueryLevelProcessor, and the attribute level
+ * of the annotation indicates which layer of memory node the query method of the
+ * QueryLevelProcessor is.
+ */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
 public @interface QueryProcessor {
-  // level of the levelProcessor
+  // level of the QueryLevelProcessor
   int level() default -1;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
index 75c472de57f55..c936bb8f395d3 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
@@ -19,19 +19,28 @@
 package org.apache.iotdb.lsm.engine;
 
 import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.lsm.request.IDeletionIRequest;
-import org.apache.iotdb.lsm.request.IInsertionIRequest;
-import org.apache.iotdb.lsm.request.IQueryIRequest;
+import org.apache.iotdb.lsm.request.IDeletionRequest;
+import org.apache.iotdb.lsm.request.IInsertionRequest;
+import org.apache.iotdb.lsm.request.IQueryRequest;
 
 import java.io.IOException;
 
+/**
+ * This interface defines the appearance of the LSM framework and provides read and write methods
+ */
 public interface ILSMEngine {
 
-   void insert(IInsertionIRequest insertionRequest);
+  /**
+   * @param insertionRequest
+   * @param 
+   * @param 
+   * @param 
+   */
+   void insert(IInsertionRequest insertionRequest);
 
-   void query(IQueryIRequest queryRequest);
+   void query(IQueryRequest queryRequest);
 
-   void delete(IDeletionIRequest deletionRequest);
+   void delete(IDeletionRequest deletionRequest);
 
   void recover();
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
index dd30c5b5caa64..fc1cdcd57cb7b 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
@@ -27,20 +27,20 @@
 import org.apache.iotdb.lsm.manager.RecoverManager;
 import org.apache.iotdb.lsm.manager.WALManager;
 import org.apache.iotdb.lsm.recover.IRecoverable;
-import org.apache.iotdb.lsm.request.IDeletionIRequest;
-import org.apache.iotdb.lsm.request.IInsertionIRequest;
-import org.apache.iotdb.lsm.request.IQueryIRequest;
+import org.apache.iotdb.lsm.request.IDeletionRequest;
+import org.apache.iotdb.lsm.request.IInsertionRequest;
+import org.apache.iotdb.lsm.request.IQueryRequest;
 import org.apache.iotdb.lsm.request.IRequest;
 
 import java.io.IOException;
 
 public class LSMEngine implements ILSMEngine, IRecoverable {
 
-  private InsertionManager insertionManager;
+  private InsertionManager insertionManager;
 
-  private DeletionManager deletionManager;
+  private DeletionManager deletionManager;
 
-  private QueryManager queryManager;
+  private QueryManager queryManager;
 
   private WALManager walManager;
 
@@ -55,17 +55,17 @@ public void setRootMemNode(T rootMemNode) {
   }
 
   @Override
-  public  void insert(IInsertionIRequest insertionRequest) {
+  public  void insert(IInsertionRequest insertionRequest) {
     insertionManager.process(rootMemNode, insertionRequest, new InsertRequestContext());
   }
 
   @Override
-  public  void query(IQueryIRequest queryRequest) {
+  public  void query(IQueryRequest queryRequest) {
     queryManager.process(rootMemNode, queryRequest, new QueryRequestContext());
   }
 
   @Override
-  public  void delete(IDeletionIRequest deletionRequest) {
+  public  void delete(IDeletionRequest deletionRequest) {
     deletionManager.process(rootMemNode, deletionRequest, new DeleteRequestContext());
   }
 
@@ -79,18 +79,18 @@ public void clear() throws IOException {
     walManager.close();
   }
 
-  protected  void setInsertionManager(
+  protected  void setInsertionManager(
       InsertionManager insertionManager) {
-    this.insertionManager = (InsertionManager) insertionManager;
+    this.insertionManager = (InsertionManager) insertionManager;
   }
 
-  protected  void setDeletionManager(
+  protected  void setDeletionManager(
       DeletionManager deletionManager) {
-    this.deletionManager = (DeletionManager) deletionManager;
+    this.deletionManager = (DeletionManager) deletionManager;
   }
 
-  protected  void setQueryManager(QueryManager queryManager) {
-    this.queryManager = (QueryManager) queryManager;
+  protected  void setQueryManager(QueryManager queryManager) {
+    this.queryManager = (QueryManager) queryManager;
   }
 
   protected WALManager getWalManager() {
@@ -109,10 +109,10 @@ protected void setRecoverManager(RecoverManager> recoverManager) {
   public  void recover(IRequest request) {
     switch (request.getRequestType()) {
       case INSERT:
-        insert((IInsertionIRequest) request);
+        insert((IInsertionRequest) request);
         break;
       case DELETE:
-        delete((IDeletionIRequest) request);
+        delete((IDeletionRequest) request);
         break;
       default:
         break;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
index 29c079b0ba205..1078ac98b8513 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
@@ -21,18 +21,31 @@
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
 import org.apache.iotdb.lsm.context.InsertRequestContext;
 import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.levelProcess.ILevelProcessor;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessorChain;
 import org.apache.iotdb.lsm.manager.DeletionManager;
 import org.apache.iotdb.lsm.manager.InsertionManager;
 import org.apache.iotdb.lsm.manager.QueryManager;
 import org.apache.iotdb.lsm.manager.RecoverManager;
 import org.apache.iotdb.lsm.manager.WALManager;
-import org.apache.iotdb.lsm.request.IDeletionIRequest;
-import org.apache.iotdb.lsm.request.IInsertionIRequest;
-import org.apache.iotdb.lsm.request.IQueryIRequest;
+import org.apache.iotdb.lsm.property.Property;
+import org.apache.iotdb.lsm.property.PropertyGenerator;
+import org.apache.iotdb.lsm.request.IDeletionRequest;
+import org.apache.iotdb.lsm.request.IInsertionRequest;
+import org.apache.iotdb.lsm.request.IQueryRequest;
+import org.apache.iotdb.lsm.request.IRequest;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.List;
 
 public class LSMEngineBuilder {
 
+  private static final Logger logger = LoggerFactory.getLogger(LSMEngineBuilder.class);
+
   private LSMEngine lsmEngine;
 
   public LSMEngineBuilder() {
@@ -44,43 +57,43 @@ public LSMEngineBuilder buildWalManager(WALManager walManager) {
     return this;
   }
 
-  public  LSMEngineBuilder buildInsertionManager(
-      LevelProcessChain levelProcessChain) {
+  public  LSMEngineBuilder buildInsertionManager(
+      LevelProcessorChain levelProcessChain) {
     InsertionManager insertionManager = new InsertionManager<>(lsmEngine.getWalManager());
     insertionManager.setLevelProcessChain(levelProcessChain);
     buildInsertionManager(insertionManager);
     return this;
   }
 
-  public  LSMEngineBuilder buildInsertionManager(
+  public  LSMEngineBuilder buildInsertionManager(
       InsertionManager insertionManager) {
     lsmEngine.setInsertionManager(insertionManager);
     return this;
   }
 
-  public  LSMEngineBuilder buildDeletionManager(
-      LevelProcessChain levelProcessChain) {
+  public  LSMEngineBuilder buildDeletionManager(
+      LevelProcessorChain levelProcessChain) {
     DeletionManager deletionManager = new DeletionManager<>(lsmEngine.getWalManager());
     deletionManager.setLevelProcessChain(levelProcessChain);
     buildDeletionManager(deletionManager);
     return this;
   }
 
-  public  LSMEngineBuilder buildDeletionManager(
+  public  LSMEngineBuilder buildDeletionManager(
       DeletionManager deletionManager) {
     lsmEngine.setDeletionManager(deletionManager);
     return this;
   }
 
-  public  LSMEngineBuilder buildQueryManager(
-      LevelProcessChain levelProcessChain) {
+  public  LSMEngineBuilder buildQueryManager(
+      LevelProcessorChain levelProcessChain) {
     QueryManager queryManager = new QueryManager<>();
     queryManager.setLevelProcessChain(levelProcessChain);
     buildQueryManager(queryManager);
     return this;
   }
 
-  public  LSMEngineBuilder buildQueryManager(
+  public  LSMEngineBuilder buildQueryManager(
       QueryManager queryManager) {
     lsmEngine.setQueryManager(queryManager);
     return this;
@@ -102,7 +115,76 @@ public LSMEngineBuilder buildRootMemNode(T rootMemNode) {
     return this;
   }
 
+  public LSMEngineBuilder buildLevelProcessors(Property property) {
+    LevelProcessorChain insertionLevelProcessChain =
+        generateLevelProcessChain(property.getInsertionLevelProcessClass());
+    LevelProcessorChain deletionLevelProcessChain =
+        generateLevelProcessChain(property.getDeletionLevelProcessClass());
+    LevelProcessorChain queryLevelProcessChain =
+        generateLevelProcessChain(property.getQueryLevelProcessClass());
+    return buildQueryManager(queryLevelProcessChain)
+        .buildInsertionManager(insertionLevelProcessChain)
+        .buildDeletionManager(deletionLevelProcessChain);
+  }
+
+  public LSMEngineBuilder buildLevelProcessors(String packageName) {
+    try {
+      Property property = PropertyGenerator.GeneratePropertyWithAnnotation(packageName);
+      buildLevelProcessors(property);
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+    return this;
+  }
+
+  public LSMEngineBuilder buildLSMManagers(Property property, WALManager walManager) {
+    try {
+      buildWalManager(walManager).buildLevelProcessors(property).buildRecoverManager();
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+    return this;
+  }
+
+  public LSMEngineBuilder buildLSMManagers(String packageName, WALManager walManager) {
+    try {
+      Property property = PropertyGenerator.GeneratePropertyWithAnnotation(packageName);
+      buildLSMManagers(property, walManager);
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+    return this;
+  }
+
   public LSMEngine build() {
     return lsmEngine;
   }
+
+  private 
+      LevelProcessorChain generateLevelProcessChain(List levelProcessClassNames) {
+    LevelProcessorChain levelProcessChain = new LevelProcessorChain<>();
+    try {
+      if (levelProcessClassNames.size() > 0) {
+        ILevelProcessor iLevelProcess =
+            levelProcessChain.nextLevel(generateLevelProcess(levelProcessClassNames.get(0)));
+        for (int i = 1; i < levelProcessClassNames.size(); i++) {
+          iLevelProcess =
+              iLevelProcess.nextLevel(generateLevelProcess(levelProcessClassNames.get(i)));
+        }
+      }
+    } catch (Exception e) {
+      logger.error(e.getMessage());
+    }
+    return levelProcessChain;
+  }
+
+  private 
+      ILevelProcessor generateLevelProcess(String className)
+          throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException,
+              InstantiationException, IllegalAccessException {
+    Class c = Class.forName(className);
+    ILevelProcessor result =
+        (ILevelProcessor) c.getDeclaredConstructor().newInstance();
+    return result;
+  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
deleted file mode 100644
index 4dc8e961f796e..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineDirector.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.lsm.engine;
-
-import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.ILevelProcessor;
-import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
-import org.apache.iotdb.lsm.manager.WALManager;
-import org.apache.iotdb.lsm.property.Property;
-import org.apache.iotdb.lsm.property.PropertyDescriptor;
-import org.apache.iotdb.lsm.request.IInsertionIRequest;
-import org.apache.iotdb.lsm.request.IQueryIRequest;
-import org.apache.iotdb.lsm.request.IRequest;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.List;
-
-public class LSMEngineDirector {
-  private static final Logger logger = LoggerFactory.getLogger(LSMEngineDirector.class);
-
-  LSMEngineBuilder lsmEngineBuilder;
-
-  public LSMEngineDirector(LSMEngineBuilder lsmEngineBuilder) {
-    this.lsmEngineBuilder = lsmEngineBuilder;
-  }
-
-  public LSMEngineDirector() {
-    lsmEngineBuilder = new LSMEngineBuilder<>();
-  }
-
-  public LSMEngine getLSMEngine(Property property, WALManager walManager) {
-    try {
-      LevelProcessChain insertionLevelProcessChain =
-          generateLevelProcessChain(property.getInsertionLevelProcessClass());
-      LevelProcessChain deletionLevelProcessChain =
-          generateLevelProcessChain(property.getDeletionLevelProcessClass());
-      LevelProcessChain queryLevelProcessChain =
-          generateLevelProcessChain(property.getQueryLevelProcessClass());
-
-      return lsmEngineBuilder
-          .buildWalManager(walManager)
-          .buildQueryManager(queryLevelProcessChain)
-          .buildInsertionManager(insertionLevelProcessChain)
-          .buildDeletionManager(deletionLevelProcessChain)
-          .buildRecoverManager()
-          .build();
-
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-    return null;
-  }
-
-  public LSMEngine getLSMEngine(String packageName, WALManager walManager) {
-    try {
-      Property property = PropertyDescriptor.getProperty(packageName);
-      return getLSMEngine(property, walManager);
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-    return null;
-  }
-
-  private 
-      LevelProcessChain generateLevelProcessChain(List levelProcessClassNames) {
-    LevelProcessChain levelProcessChain = new LevelProcessChain<>();
-    try {
-      if (levelProcessClassNames.size() > 0) {
-        ILevelProcessor iLevelProcess =
-            levelProcessChain.nextLevel(generateLevelProcess(levelProcessClassNames.get(0)));
-        for (int i = 1; i < levelProcessClassNames.size(); i++) {
-          iLevelProcess =
-              iLevelProcess.nextLevel(generateLevelProcess(levelProcessClassNames.get(i)));
-        }
-      }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
-    }
-    return levelProcessChain;
-  }
-
-  private 
-      ILevelProcessor generateLevelProcess(String className)
-          throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException,
-              InstantiationException, IllegalAccessException {
-    Class c = Class.forName(className);
-    ILevelProcessor result =
-        (ILevelProcessor) c.getDeclaredConstructor().newInstance();
-    return result;
-  }
-}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java
similarity index 94%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java
index 942fd82abac89..2f035478d4f36 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessChain.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java
@@ -21,7 +21,7 @@
 import org.apache.iotdb.lsm.context.RequestContext;
 import org.apache.iotdb.lsm.request.IRequest;
 
-public class LevelProcessChain {
+public class LevelProcessorChain {
 
   // the level process of the first layer of memory nodes
   ILevelProcessor headLevelProcess;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
index 13f55dd425752..9dccfec9e3431 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.context.RequestContext;
-import org.apache.iotdb.lsm.levelProcess.LevelProcessChain;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessorChain;
 import org.apache.iotdb.lsm.request.IRequest;
 
 /** basic lsm manager */
@@ -27,11 +27,11 @@ public abstract class BasicLSMManager {
 
   // the level process of the first layer of memory nodes
-  LevelProcessChain levelProcessChain;
+  LevelProcessorChain levelProcessChain;
 
   public BasicLSMManager() {}
 
-  public BasicLSMManager(LevelProcessChain levelProcessChain) {
+  public BasicLSMManager(LevelProcessorChain levelProcessChain) {
     this.levelProcessChain = levelProcessChain;
   }
 
@@ -49,7 +49,7 @@ public void process(T root, R request, C context) {
     postProcess(root, request, context);
   }
 
-  public void setLevelProcessChain(LevelProcessChain levelProcessChain) {
+  public void setLevelProcessChain(LevelProcessorChain levelProcessChain) {
     this.levelProcessChain = levelProcessChain;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
index 17286d3b968b1..0519c1a40eb29 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
@@ -19,10 +19,10 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.request.IDeletionIRequest;
+import org.apache.iotdb.lsm.request.IDeletionRequest;
 
 /** manage deletion to MemTable */
-public class DeletionManager
+public class DeletionManager
     extends BasicLSMManager {
 
   // use wal manager object to write wal file on deletion
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
index 3855ea1bc19b0..77fb53e3fe38e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
@@ -19,10 +19,10 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.request.IInsertionIRequest;
+import org.apache.iotdb.lsm.request.IInsertionRequest;
 
 /** manage insertion to MemTable */
-public class InsertionManager
+public class InsertionManager
     extends BasicLSMManager {
 
   // use wal manager object to write wal file on insertion
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
index fd1055ed0521d..1d905e8afed39 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
@@ -19,10 +19,10 @@
 package org.apache.iotdb.lsm.manager;
 
 import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.request.IQueryIRequest;
+import org.apache.iotdb.lsm.request.IQueryRequest;
 
 /** manage query to MemTable */
-public class QueryManager
+public class QueryManager
     extends BasicLSMManager {
   @Override
   public void preProcess(T root, R request, QueryRequestContext context) {}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyGenerator.java
similarity index 97%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyGenerator.java
index e40ac7af97e23..647c5759ac5e4 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyDescriptor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyGenerator.java
@@ -31,9 +31,9 @@
 import java.util.List;
 import java.util.Set;
 
-public class PropertyDescriptor {
+public class PropertyGenerator {
 
-  public static Property getProperty(String packageName) throws Exception {
+  public static Property GeneratePropertyWithAnnotation(String packageName) {
     Reflections reflections =
         new Reflections(
             new ConfigurationBuilder()
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionIRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java
similarity index 93%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionIRequest.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java
index c6273a96c526b..24f1e58f85bcf 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionIRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.request;
 
-public interface IDeletionIRequest extends IRequest {
+public interface IDeletionRequest extends IRequest {
 
   RequestType requestType = RequestType.DELETE;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionIRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java
similarity index 93%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionIRequest.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java
index 3ae637503d40f..46d029519bf89 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionIRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.request;
 
-public interface IInsertionIRequest extends IRequest {
+public interface IInsertionRequest extends IRequest {
 
   RequestType requestType = RequestType.INSERT;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryIRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java
similarity index 93%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryIRequest.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java
index 2b5d14431808e..d6fdf6369dee7 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryIRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.request;
 
-public interface IQueryIRequest extends IRequest {
+public interface IQueryRequest extends IRequest {
 
   RequestType requestType = RequestType.QUERY;
 

From 41e49eeaf51ba6e6ebc6380785a86e4d56bd12d9 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Tue, 25 Oct 2022 11:01:29 +0800
Subject: [PATCH 49/53] add javadoc

---
 .../tagIndex/ITagInvertedIndex.java           |  11 ++
 .../tagIndex/Request/DeletionRequest.java     |   6 +-
 .../tagIndex/Request/InsertionRequest.java    |   6 +-
 .../tagIndex/Request/QueryRequest.java        |   5 +-
 .../tagIndex/Request/RequestStatus.java       |   1 +
 .../tagIndex/TagInvertedIndex.java            |  58 ++++++-
 .../tagIndex/deletion/MemChunkDeletion.java   |   2 +-
 .../deletion/MemChunkGroupDeletion.java       |   2 +-
 .../tagIndex/deletion/MemTableDeletion.java   |   2 +-
 .../deletion/MemTableGroupDeletion.java       |  17 +-
 .../insertion/MemChunkGroupInsertion.java     |   2 +-
 .../tagIndex/insertion/MemChunkInsertion.java |   2 +-
 .../insertion/MemTableGroupInsertion.java     |  16 +-
 .../tagIndex/insertion/MemTableInsertion.java |   2 +-
 .../tagIndex/memtable/MemTableGroup.java      |   2 +-
 .../tagIndex/query/MemChunkGroupQuery.java    |   2 +-
 .../tagIndex/query/MemChunkQuery.java         |   2 +-
 .../tagIndex/query/MemTableGroupQuery.java    |  17 +-
 .../tagIndex/query/MemTableQuery.java         |   2 +-
 .../ApplicationContext.java}                  |  11 +-
 .../ApplicationContextGenerator.java}         |  71 +++++---
 .../DeleteRequestContext.java                 |   2 +-
 .../FlushRequestContext.java                  |   2 +-
 .../InsertRequestContext.java                 |   2 +-
 .../QueryRequestContext.java                  |   2 +-
 .../{ => requestcontext}/RequestContext.java  |   2 +-
 .../apache/iotdb/lsm/engine/ILSMEngine.java   |  33 +++-
 .../lsm/{recover => engine}/IRecoverable.java |  13 +-
 .../apache/iotdb/lsm/engine/LSMEngine.java    |  91 +++++++---
 .../iotdb/lsm/engine/LSMEngineBuilder.java    | 164 ++++++++++++++----
 .../lsm/levelProcess/BasicLevelProcessor.java |   2 +-
 .../levelProcess/DeleteLevelProcessor.java    |   2 +-
 .../lsm/levelProcess/FlushLevelProcessor.java |   2 +-
 .../lsm/levelProcess/ILevelProcessor.java     |   2 +-
 .../levelProcess/InsertLevelProcessor.java    |   2 +-
 .../lsm/levelProcess/LevelProcessorChain.java |  10 +-
 .../lsm/levelProcess/QueryLevelProcessor.java |   2 +-
 .../iotdb/lsm/manager/BasicLSMManager.java    |  18 +-
 .../iotdb/lsm/manager/DeletionManager.java    |   5 +-
 .../apache/iotdb/lsm/manager/ILSMManager.java |  17 +-
 .../iotdb/lsm/manager/InsertionManager.java   |   5 +-
 .../iotdb/lsm/manager/QueryManager.java       |   4 +-
 .../iotdb/lsm/manager/RecoverManager.java     |   4 +-
 .../apache/iotdb/lsm/manager/WALManager.java  |  20 +--
 .../iotdb/lsm/request/IDeletionRequest.java   |   3 +-
 .../iotdb/lsm/request/IInsertionRequest.java  |   3 +-
 .../iotdb/lsm/request/IQueryRequest.java      |   6 +
 .../apache/iotdb/lsm/request/IRequest.java    |  42 ++++-
 .../apache/iotdb/lsm/request/RequestType.java |   1 +
 .../iotdb/lsm/strategy/BFSAccessStrategy.java |   2 +-
 .../iotdb/lsm/strategy/IAccessStrategy.java   |   2 +-
 .../lsm/strategy/PostOrderAccessStrategy.java |   2 +-
 .../lsm/strategy/PreOrderAccessStrategy.java  |   2 +-
 .../lsm/strategy/RBFSAccessStrategy.java      |   2 +-
 54 files changed, 552 insertions(+), 158 deletions(-)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/{property/Property.java => context/applicationcontext/ApplicationContext.java} (77%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/{property/PropertyGenerator.java => context/applicationcontext/ApplicationContextGenerator.java} (52%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/{ => requestcontext}/DeleteRequestContext.java (95%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/{ => requestcontext}/FlushRequestContext.java (95%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/{ => requestcontext}/InsertRequestContext.java (95%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/{ => requestcontext}/QueryRequestContext.java (95%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/{ => requestcontext}/RequestContext.java (97%)
 rename schema-engine-tag/src/main/java/org/apache/iotdb/lsm/{recover => engine}/IRecoverable.java (65%)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java
index aaec01847cd6d..9dab290d900b3 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java
@@ -18,6 +18,9 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex;
 
+import org.apache.iotdb.commons.utils.TestOnly;
+
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -47,4 +50,12 @@ public interface ITagInvertedIndex {
    * @return device ids
    */
   List getMatchedIDs(Map tags);
+
+  /**
+   * Close all open resources
+   *
+   * @throws IOException
+   */
+  @TestOnly
+  void clear() throws IOException;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
index 6d2e1b21f6ab1..632be8190e18c 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
@@ -18,17 +18,21 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.request.IDeletionRequest;
 
 import java.util.List;
 
+/** Represents a deletion request */
 public class DeletionRequest implements IDeletionRequest {
 
+  // tags
   List keys;
 
+  // int32 id
   int value;
 
+  // deletion result
   RequestStatus requestStatus;
 
   public DeletionRequest(List keys, int value) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
index b97c66b2dbd70..e087917d4356a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
@@ -18,17 +18,21 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.request.IInsertionRequest;
 
 import java.util.List;
 
+/** Represents a insertion request */
 public class InsertionRequest implements IInsertionRequest {
 
+  // tags
   List keys;
 
+  // int32 id
   int value;
 
+  // insertion result
   RequestStatus requestStatus;
 
   public InsertionRequest() {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
index 7cd72d749e289..47dd86274462d 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
@@ -18,17 +18,20 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.request.IQueryRequest;
 
 import org.roaringbitmap.RoaringBitmap;
 
 import java.util.List;
 
+/** Represents a query request */
 public class QueryRequest implements IQueryRequest {
 
+  // tags
   List keys;
 
+  // Query results, encapsulating ids
   RoaringBitmap roaringBitmap;
 
   public QueryRequest(List keys) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java
index 185ecd7bd2389..fc81aed24c603 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
 
+/** Indicates the status of the request, whether it is successful after the request is processed */
 public class RequestStatus {
 
   Boolean success;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index 8b641e09265b5..917639b55dbbb 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -41,17 +41,27 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+/** tag inverted index, tag is  and id is int32 auto increment id */
 public class TagInvertedIndex implements ITagInvertedIndex {
 
+  // This file records the wal log
   private static final String WAL_FILE_NAME = "tag_inverted_index.log";
 
   private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class);
 
+  // Manage configuration information of tag schema region
   private static final TagSchemaConfig tagSchemaConfig =
       TagSchemaDescriptor.getInstance().getTagSchemaConfig();
 
+  // Directly use the lsm engine that comes with the lsm framework to implement the tag inverted
+  // index
   LSMEngine lsmEngine;
 
+  /**
+   * initialization method
+   *
+   * @param schemaDirPath schema dirPath
+   */
   public TagInvertedIndex(String schemaDirPath) {
     try {
       WALManager walManager =
@@ -61,20 +71,30 @@ public TagInvertedIndex(String schemaDirPath) {
               tagSchemaConfig.getWalBufferSize(),
               new WALEntry(),
               false);
+      // root memory node, used to manage working and immutableMemTables
       MemTableGroup memTableGroup =
           new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable());
-      LSMEngineBuilder lsmEngineBuilder = new LSMEngineBuilder<>();
+
+      // build lsm engine
       lsmEngine =
-          lsmEngineBuilder
+          new LSMEngineBuilder()
               .buildLSMManagers("org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex", walManager)
               .buildRootMemNode(memTableGroup)
               .build();
+
+      // recover the lsm engine
       lsmEngine.recover();
     } catch (Exception e) {
       logger.error(e.getMessage());
     }
   }
 
+  /**
+   * insert tags and device id
+   *
+   * @param tags tags like: 
+   * @param id INT32 device id
+   */
   @Override
   public synchronized void addTags(Map tags, int id) {
     try {
@@ -88,6 +108,12 @@ public synchronized void addTags(Map tags, int id) {
     }
   }
 
+  /**
+   * delete tags and id using delete request context
+   *
+   * @param tags tags like: 
+   * @param id INT32 device id
+   */
   @Override
   public synchronized void removeTags(Map tags, int id) {
     try {
@@ -101,6 +127,12 @@ public synchronized void removeTags(Map tags, int id) {
     }
   }
 
+  /**
+   * get all matching device ids
+   *
+   * @param tags tags like: 
+   * @return
+   */
   @Override
   public synchronized List getMatchedIDs(Map tags) {
     RoaringBitmap roaringBitmap = new RoaringBitmap();
@@ -121,6 +153,13 @@ public synchronized List getMatchedIDs(Map tags) {
     return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList());
   }
 
+  /**
+   * Generate the keys in the request
+   *
+   * @param tagKey tag key
+   * @param tagValue tag value
+   * @return keys
+   */
   private List generateKeys(String tagKey, String tagValue) {
     List keys = new ArrayList<>();
     keys.add(tagKey);
@@ -128,12 +167,25 @@ private List generateKeys(String tagKey, String tagValue) {
     return keys;
   }
 
-  private RoaringBitmap getMatchedIDs(String tagKey, String tagValue) throws Exception {
+  /**
+   * Get ids matching the tag
+   *
+   * @param tagKey tag key
+   * @param tagValue tag value
+   * @return roaring bitmap
+   */
+  private RoaringBitmap getMatchedIDs(String tagKey, String tagValue) {
     QueryRequest queryRequest = new QueryRequest(generateKeys(tagKey, tagValue));
     lsmEngine.query(queryRequest);
     return queryRequest.getResult();
   }
 
+  /**
+   * Close all open resources
+   *
+   * @throws IOException
+   */
+  @Override
   @TestOnly
   public void clear() throws IOException {
     lsmEngine.clear();
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
index 6520ec8bb74a3..7e0d70d0239ab 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java
@@ -21,7 +21,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.lsm.annotation.DeletionProcessor;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext;
 import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor;
 
 import java.util.List;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
index c00dafa140843..c6718d941739f 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java
@@ -22,7 +22,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.lsm.annotation.DeletionProcessor;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext;
 import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor;
 
 import java.util.ArrayList;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
index 25ec70c5cf93a..fa27bba523f88 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java
@@ -22,7 +22,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.lsm.annotation.DeletionProcessor;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext;
 import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor;
 
 import java.util.ArrayList;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
index bb9d1573c455f..249dd9acfb5b5 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java
@@ -22,15 +22,24 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
 import org.apache.iotdb.lsm.annotation.DeletionProcessor;
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext;
 import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 
+/** deletion for MemTableGroup */
 @DeletionProcessor(level = 0)
 public class MemTableGroupDeletion
     extends DeleteLevelProcessor {
+
+  /**
+   * get all MemTables that need to be processed in the current MemTableGroup
+   *
+   * @param memNode memory node
+   * @param context request context
+   * @return A list of saved MemTables
+   */
   @Override
   public List getChildren(
       MemTableGroup memNode, DeletionRequest request, DeleteRequestContext context) {
@@ -45,6 +54,12 @@ public List getChildren(
     return memTables;
   }
 
+  /**
+   * the delete method corresponding to the MemTableGroup node, do nothing
+   *
+   * @param memNode memory node
+   * @param context deletion request context
+   */
   @Override
   public void delete(
       MemTableGroup memNode, DeletionRequest request, DeleteRequestContext context) {}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
index 07a58b4483ebb..9abf201dc967c 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java
@@ -22,7 +22,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.lsm.annotation.InsertionProcessor;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext;
 import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor;
 
 import java.util.ArrayList;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
index c80793dc603f7..5dd91ef869f32 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java
@@ -21,7 +21,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.lsm.annotation.InsertionProcessor;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext;
 import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor;
 
 import java.util.List;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
index bef19b9532e6b..46507443b91f7 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java
@@ -23,17 +23,25 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
 import org.apache.iotdb.lsm.annotation.InsertionProcessor;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext;
 import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+/** insertion for MemTableGroup */
 @InsertionProcessor(level = 0)
 public class MemTableGroupInsertion
     extends InsertLevelProcessor {
 
+  /**
+   * get all MemTable that need to be processed in the current MemTableGroup
+   *
+   * @param memNode memory node
+   * @param context request context
+   * @return A list of saved MemTables
+   */
   @Override
   public List getChildren(
       MemTableGroup memNode, InsertionRequest request, InsertRequestContext context) {
@@ -42,6 +50,12 @@ public List getChildren(
     return memTables;
   }
 
+  /**
+   * the insert method corresponding to the MemTableGroup node
+   *
+   * @param memNode memory node
+   * @param context insert request context
+   */
   @Override
   public void insert(
       MemTableGroup memNode, InsertionRequest request, InsertRequestContext context) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
index 16b4c7f4cc44a..7b20522ae78f9 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java
@@ -22,7 +22,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.lsm.annotation.InsertionProcessor;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext;
 import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor;
 
 import java.util.ArrayList;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java
index be7e3e9a6ae13..e49ce546a46f5 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java
@@ -21,7 +21,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-/** used to manage working and immutableMemTable */
+/** used to manage working and immutableMemTables */
 public class MemTableGroup {
 
   // the maximum number of device ids managed by a working memTable
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
index a6001f425f808..62636914d58c1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java
@@ -22,7 +22,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.lsm.annotation.QueryProcessor;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor;
 
 import java.util.ArrayList;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
index 96d389f7520df..c95e5accbb34e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
@@ -21,7 +21,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
 import org.apache.iotdb.lsm.annotation.QueryProcessor;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor;
 
 import org.roaringbitmap.RoaringBitmap;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
index 96af3f18da691..39e09db7f9603 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java
@@ -22,14 +22,23 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
 import org.apache.iotdb.lsm.annotation.QueryProcessor;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
 
+/** query for MemTableGroup */
 @QueryProcessor(level = 0)
 public class MemTableGroupQuery extends QueryLevelProcessor {
+
+  /**
+   * get all MemTable that need to be processed in the current MemTableGroup
+   *
+   * @param memNode memory node
+   * @param context request context
+   * @return A list of saved MemTable
+   */
   @Override
   public List getChildren(
       MemTableGroup memNode, QueryRequest request, QueryRequestContext context) {
@@ -39,6 +48,12 @@ public List getChildren(
     return memTables;
   }
 
+  /**
+   * the query method corresponding to the MemTable node, do nothing
+   *
+   * @param memNode memory node
+   * @param context query request context
+   */
   @Override
   public void query(MemTableGroup memNode, QueryRequest request, QueryRequestContext context) {}
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
index ff5d6c00c2e3f..4fb1f42506714 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
@@ -22,7 +22,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable;
 import org.apache.iotdb.lsm.annotation.QueryProcessor;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor;
 
 import org.roaringbitmap.RoaringBitmap;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/Property.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContext.java
similarity index 77%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/Property.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContext.java
index 531ad0d245514..8a7f6646147f4 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/Property.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContext.java
@@ -16,16 +16,23 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.lsm.property;
+package org.apache.iotdb.lsm.context.applicationcontext;
 
 import java.util.List;
 
-public class Property {
+/**
+ * Indicates the information of a user program, the user can customize the configuration startup
+ * information, and the lsm framework can use this information to generate the lsm engine
+ */
+public class ApplicationContext {
 
+  // Save the insertion level processor of each layer in hierarchical order
   List insertionLevelProcessClass;
 
+  // Save the deletion level processor of each layer in hierarchical order
   List deletionLevelProcessClass;
 
+  // Save the query level processor of each layer in hierarchical order
   List queryLevelProcessClass;
 
   public List getInsertionLevelProcessClass() {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyGenerator.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContextGenerator.java
similarity index 52%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyGenerator.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContextGenerator.java
index 647c5759ac5e4..c1bbfc5571609 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/property/PropertyGenerator.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContextGenerator.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.lsm.property;
+package org.apache.iotdb.lsm.context.applicationcontext;
 
 import org.apache.iotdb.lsm.annotation.DeletionProcessor;
 import org.apache.iotdb.lsm.annotation.InsertionProcessor;
@@ -31,60 +31,89 @@
 import java.util.List;
 import java.util.Set;
 
-public class PropertyGenerator {
+/** Used to generate ApplicationContext object based on annotations or configuration files */
+public class ApplicationContextGenerator {
 
-  public static Property GeneratePropertyWithAnnotation(String packageName) {
+  /**
+   * Scan the package to get all classes, and generate ApplicationContext object based on the
+   * annotations of these classes
+   *
+   * @param packageName package name
+   * @return ApplicationContext object
+   */
+  public static ApplicationContext GeneratePropertyWithAnnotation(String packageName) {
     Reflections reflections =
         new Reflections(
             new ConfigurationBuilder()
                 .forPackage(packageName)
                 .filterInputsBy(new FilterBuilder().includePackage(packageName)));
-    Property property = new Property();
-    setDeletionLevelProcess(property, reflections);
-    setInsertionLevelProcess(property, reflections);
-    setQueryLevelProcess(property, reflections);
-    return property;
+    ApplicationContext applicationContext = new ApplicationContext();
+    setDeletionLevelProcessor(applicationContext, reflections);
+    setInsertionLevelProcessor(applicationContext, reflections);
+    setQueryLevelProcessor(applicationContext, reflections);
+    return applicationContext;
   }
 
-  private static void setInsertionLevelProcess(Property property, Reflections reflections) {
+  /**
+   * Assign value to the insertion level processor of the ApplicationContext object
+   *
+   * @param applicationContext ApplicationContext object
+   * @param reflections This object holds all the classes scanned in the package
+   */
+  private static void setInsertionLevelProcessor(
+      ApplicationContext applicationContext, Reflections reflections) {
     Set> annotated = reflections.getTypesAnnotatedWith(InsertionProcessor.class);
     List levelProcessClass = new ArrayList<>();
     for (Class clz : annotated) {
       InsertionProcessor annotationInfo = clz.getAnnotation(InsertionProcessor.class);
       setLevelProcessors(levelProcessClass, clz, annotationInfo.level());
     }
-    property.setInsertionLevelProcessClass(levelProcessClass);
+    applicationContext.setInsertionLevelProcessClass(levelProcessClass);
   }
 
-  private static void setDeletionLevelProcess(Property property, Reflections reflections) {
+  /**
+   * Assign value to the deletion level processor of the ApplicationContext object
+   *
+   * @param applicationContext ApplicationContext object
+   * @param reflections This object holds all the classes scanned in the package
+   */
+  private static void setDeletionLevelProcessor(
+      ApplicationContext applicationContext, Reflections reflections) {
     Set> annotated = reflections.getTypesAnnotatedWith(DeletionProcessor.class);
     List levelProcessClass = new ArrayList<>();
     for (Class clz : annotated) {
       DeletionProcessor annotationInfo = clz.getAnnotation(DeletionProcessor.class);
       setLevelProcessors(levelProcessClass, clz, annotationInfo.level());
     }
-    property.setDeletionLevelProcessClass(levelProcessClass);
+    applicationContext.setDeletionLevelProcessClass(levelProcessClass);
   }
 
-  private static  void setQueryLevelProcess(
-      Property property, Reflections reflections) {
+  /**
+   * Assign value to the query level processor of the ApplicationContext object
+   *
+   * @param applicationContext ApplicationContext object
+   * @param reflections This object holds all the classes scanned in the package
+   */
+  private static  void setQueryLevelProcessor(
+      ApplicationContext applicationContext, Reflections reflections) {
     List levelProcessClass = new ArrayList<>();
     Set> annotated = reflections.getTypesAnnotatedWith(QueryProcessor.class);
     for (Class clz : annotated) {
       QueryProcessor annotationInfo = clz.getAnnotation(QueryProcessor.class);
       setLevelProcessors(levelProcessClass, clz, annotationInfo.level());
     }
-    property.setQueryLevelProcessClass(levelProcessClass);
+    applicationContext.setQueryLevelProcessClass(levelProcessClass);
   }
 
-  private static void setLevelProcessors(List levelProcessClass, Class clz, int level) {
-    if (level < levelProcessClass.size()) {
-      levelProcessClass.set(level, clz.getName());
+  private static void setLevelProcessors(
+      List levelProcessorClass, Class clz, int level) {
+    if (level < levelProcessorClass.size()) {
+      levelProcessorClass.set(level, clz.getName());
     } else {
-      for (int i = levelProcessClass.size(); i < level; i++) {
-        levelProcessClass.add("");
+      for (int i = levelProcessorClass.size(); i < level; i++) {
+        levelProcessorClass.add("");
       }
-      levelProcessClass.add(clz.getName());
+      levelProcessorClass.add(clz.getName());
     }
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/DeleteRequestContext.java
similarity index 95%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/DeleteRequestContext.java
index 1386d0d3c01f7..1d996680b1c90 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/DeleteRequestContext.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.lsm.context;
+package org.apache.iotdb.lsm.context.requestcontext;
 
 import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/FlushRequestContext.java
similarity index 95%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/FlushRequestContext.java
index 4a66ec82e7fb9..29e0ed1b9c963 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/FlushRequestContext.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.lsm.context;
+package org.apache.iotdb.lsm.context.requestcontext;
 
 import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/InsertRequestContext.java
similarity index 95%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/InsertRequestContext.java
index 0555d35ec5dfb..d519f6994e907 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/InsertRequestContext.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.lsm.context;
+package org.apache.iotdb.lsm.context.requestcontext;
 
 import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/QueryRequestContext.java
similarity index 95%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/QueryRequestContext.java
index 2f888571f024f..7534dee516fb4 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/QueryRequestContext.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.lsm.context;
+package org.apache.iotdb.lsm.context.requestcontext;
 
 import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java
similarity index 97%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java
index ee0d3a6f88b72..607edd3698d09 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.lsm.context;
+package org.apache.iotdb.lsm.context.requestcontext;
 
 import org.apache.iotdb.lsm.strategy.IAccessStrategy;
 import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
index c936bb8f395d3..88f473815ea41 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
@@ -28,22 +28,45 @@
 /**
  * This interface defines the appearance of the LSM framework and provides read and write methods
  */
-public interface ILSMEngine {
+public interface ILSMEngine extends IRecoverable {
 
   /**
-   * @param insertionRequest
-   * @param 
-   * @param 
-   * @param 
+   * Use this ILSMEngine to insert data
+   *
+   * @param insertionRequest Encapsulates the data to be inserted
+   * @param  The type of key in the request data
+   * @param  The type of value in the request data
+   * @param  return value type after insertion
    */
    void insert(IInsertionRequest insertionRequest);
 
+  /**
+   * Use this ILSMEngine to query
+   *
+   * @param queryRequest Encapsulates query data
+   * @param  The type of key in the request data
+   * @param  return value type after query
+   */
    void query(IQueryRequest queryRequest);
 
+  /**
+   * Use this ILSMEngine to delete data
+   *
+   * @param deletionRequest Encapsulates the data to be deleted
+   * @param  The type of key in the request data
+   * @param  The type of value in the request data
+   * @param  return value type after deletion
+   */
    void delete(IDeletionRequest deletionRequest);
 
+  /** recover the ILSMEngine */
   void recover();
 
+  /**
+   * Close all open resources
+   *
+   * @throws IOException
+   */
   @TestOnly
   void clear() throws IOException;
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java
similarity index 65%
rename from schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
rename to schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java
index e82905b795f39..752ae534d9122 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/recover/IRecoverable.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java
@@ -16,11 +16,20 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.lsm.recover;
+package org.apache.iotdb.lsm.engine;
 
 import org.apache.iotdb.lsm.request.IRequest;
 
+/** Any object implements this interface can be recovered by RecoverManager */
 public interface IRecoverable {
 
-   void recover(IRequest Request);
+  /**
+   * Use Request to recover the object which implements the interface
+   *
+   * @param request insertionRequest or deletionRequest
+   * @param  The type of key in the request data
+   * @param  The type of value in the request data
+   * @param  return value type after recover
+   */
+   void recover(IRequest request);
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
index fc1cdcd57cb7b..58c5623c2a6de 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
@@ -18,15 +18,15 @@
  */
 package org.apache.iotdb.lsm.engine;
 
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext;
 import org.apache.iotdb.lsm.manager.DeletionManager;
 import org.apache.iotdb.lsm.manager.InsertionManager;
 import org.apache.iotdb.lsm.manager.QueryManager;
 import org.apache.iotdb.lsm.manager.RecoverManager;
 import org.apache.iotdb.lsm.manager.WALManager;
-import org.apache.iotdb.lsm.recover.IRecoverable;
 import org.apache.iotdb.lsm.request.IDeletionRequest;
 import org.apache.iotdb.lsm.request.IInsertionRequest;
 import org.apache.iotdb.lsm.request.IQueryRequest;
@@ -34,51 +34,110 @@
 
 import java.io.IOException;
 
-public class LSMEngine implements ILSMEngine, IRecoverable {
+/**
+ * The default ILSMEngine implementation class provided by the LSM framework
+ *
+ * @param  The type of root memory node handled by this engine
+ */
+public class LSMEngine implements ILSMEngine {
 
+  // Use the framework's default InsertionManager object to handle insert requests
   private InsertionManager insertionManager;
 
+  // Use the framework's default DeletionManager object to handle delete requests
   private DeletionManager deletionManager;
 
+  // Use the framework's default QueryManager object to handle query requests
   private QueryManager queryManager;
 
+  // Used to manage wal logs
   private WALManager walManager;
 
+  // Use the framework's default RecoverManager object to recover the LSMEngine
   private RecoverManager> recoverManager;
 
+  // Managed root memory node
   private T rootMemNode;
 
   public LSMEngine() {}
 
-  public void setRootMemNode(T rootMemNode) {
-    this.rootMemNode = rootMemNode;
-  }
-
+  /**
+   * Use this LSMEngine to insert data
+   *
+   * @param insertionRequest Encapsulates the data to be inserted
+   * @param  The type of key in the request data
+   * @param  The type of value in the request data
+   * @param  return value type after insertion
+   */
   @Override
   public  void insert(IInsertionRequest insertionRequest) {
     insertionManager.process(rootMemNode, insertionRequest, new InsertRequestContext());
   }
 
+  /**
+   * Use this LSMEngine to query
+   *
+   * @param queryRequest Encapsulates query data
+   * @param  The type of key in the request data
+   * @param  return value type after query
+   */
   @Override
   public  void query(IQueryRequest queryRequest) {
     queryManager.process(rootMemNode, queryRequest, new QueryRequestContext());
   }
 
+  /**
+   * Use this LSMEngine to delete data
+   *
+   * @param deletionRequest Encapsulates the data to be deleted
+   * @param  The type of key in the request data
+   * @param  The type of value in the request data
+   * @param  return value type after deletion
+   */
   @Override
   public  void delete(IDeletionRequest deletionRequest) {
     deletionManager.process(rootMemNode, deletionRequest, new DeleteRequestContext());
   }
 
+  /** recover the LSMEngine */
   @Override
   public void recover() {
     recoverManager.recover(this);
   }
 
+  /**
+   * Close all open resources
+   *
+   * @throws IOException
+   */
   @Override
+  @TestOnly
   public void clear() throws IOException {
     walManager.close();
   }
 
+  /**
+   * Use Request to recover the LSMEngine
+   *
+   * @param request insertionRequest or deletionRequest
+   * @param  The type of key in the request data
+   * @param  The type of value in the request data
+   * @param  return value type after deletion
+   */
+  @Override
+  public  void recover(IRequest request) {
+    switch (request.getRequestType()) {
+      case INSERT:
+        insert((IInsertionRequest) request);
+        break;
+      case DELETE:
+        delete((IDeletionRequest) request);
+        break;
+      default:
+        break;
+    }
+  }
+
   protected  void setInsertionManager(
       InsertionManager insertionManager) {
     this.insertionManager = (InsertionManager) insertionManager;
@@ -105,17 +164,7 @@ protected void setRecoverManager(RecoverManager> recoverManager) {
     this.recoverManager = recoverManager;
   }
 
-  @Override
-  public  void recover(IRequest request) {
-    switch (request.getRequestType()) {
-      case INSERT:
-        insert((IInsertionRequest) request);
-        break;
-      case DELETE:
-        delete((IDeletionRequest) request);
-        break;
-      default:
-        break;
-    }
+  protected void setRootMemNode(T rootMemNode) {
+    this.rootMemNode = rootMemNode;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
index 1078ac98b8513..580575277331c 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java
@@ -18,10 +18,12 @@
  */
 package org.apache.iotdb.lsm.engine;
 
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
-import org.apache.iotdb.lsm.context.InsertRequestContext;
-import org.apache.iotdb.lsm.context.QueryRequestContext;
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.applicationcontext.ApplicationContext;
+import org.apache.iotdb.lsm.context.applicationcontext.ApplicationContextGenerator;
+import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.levelProcess.ILevelProcessor;
 import org.apache.iotdb.lsm.levelProcess.LevelProcessorChain;
 import org.apache.iotdb.lsm.manager.DeletionManager;
@@ -29,8 +31,6 @@
 import org.apache.iotdb.lsm.manager.QueryManager;
 import org.apache.iotdb.lsm.manager.RecoverManager;
 import org.apache.iotdb.lsm.manager.WALManager;
-import org.apache.iotdb.lsm.property.Property;
-import org.apache.iotdb.lsm.property.PropertyGenerator;
 import org.apache.iotdb.lsm.request.IDeletionRequest;
 import org.apache.iotdb.lsm.request.IInsertionRequest;
 import org.apache.iotdb.lsm.request.IQueryRequest;
@@ -42,94 +42,153 @@
 import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 
+/**
+ * Build the LSMEngine object
+ *
+ * @param  The type of root memory node handled by this engine
+ */
 public class LSMEngineBuilder {
 
   private static final Logger logger = LoggerFactory.getLogger(LSMEngineBuilder.class);
 
+  // The constructed LSMEngine object
   private LSMEngine lsmEngine;
 
   public LSMEngineBuilder() {
     lsmEngine = new LSMEngine<>();
   }
 
+  /**
+   * build WalManager for lsmEngine
+   *
+   * @param walManager WalManager object
+   */
   public LSMEngineBuilder buildWalManager(WALManager walManager) {
     lsmEngine.setWalManager(walManager);
     return this;
   }
 
+  /**
+   * build InsertionManager for lsmEngine
+   *
+   * @param levelProcessChain insert level processors chain
+   * @param  extends IInsertionRequest
+   */
   public  LSMEngineBuilder buildInsertionManager(
       LevelProcessorChain levelProcessChain) {
     InsertionManager insertionManager = new InsertionManager<>(lsmEngine.getWalManager());
-    insertionManager.setLevelProcessChain(levelProcessChain);
+    insertionManager.setLevelProcessorsChain(levelProcessChain);
     buildInsertionManager(insertionManager);
     return this;
   }
 
+  /**
+   * build InsertionManager for lsmEngine
+   *
+   * @param insertionManager InsertionManager object
+   * @param  extends IInsertionRequest
+   */
   public  LSMEngineBuilder buildInsertionManager(
       InsertionManager insertionManager) {
     lsmEngine.setInsertionManager(insertionManager);
     return this;
   }
 
+  /**
+   * build DeletionManager for lsmEngine
+   *
+   * @param levelProcessChain delete level processors chain
+   * @param  extends IDeletionRequest
+   */
   public  LSMEngineBuilder buildDeletionManager(
       LevelProcessorChain levelProcessChain) {
     DeletionManager deletionManager = new DeletionManager<>(lsmEngine.getWalManager());
-    deletionManager.setLevelProcessChain(levelProcessChain);
+    deletionManager.setLevelProcessorsChain(levelProcessChain);
     buildDeletionManager(deletionManager);
     return this;
   }
 
+  /**
+   * build DeletionManager for lsmEngine
+   *
+   * @param deletionManager DeletionManager object
+   * @param  extends IDeletionRequest
+   */
   public  LSMEngineBuilder buildDeletionManager(
       DeletionManager deletionManager) {
     lsmEngine.setDeletionManager(deletionManager);
     return this;
   }
 
+  /**
+   * build QueryManager for lsmEngine
+   *
+   * @param levelProcessChain query level processors chain
+   * @param  extends IQueryRequest
+   */
   public  LSMEngineBuilder buildQueryManager(
       LevelProcessorChain levelProcessChain) {
     QueryManager queryManager = new QueryManager<>();
-    queryManager.setLevelProcessChain(levelProcessChain);
+    queryManager.setLevelProcessorsChain(levelProcessChain);
     buildQueryManager(queryManager);
     return this;
   }
 
+  /**
+   * build QueryManager for lsmEngine
+   *
+   * @param queryManager QueryManager object
+   * @param  extends IQueryRequest
+   */
   public  LSMEngineBuilder buildQueryManager(
       QueryManager queryManager) {
     lsmEngine.setQueryManager(queryManager);
     return this;
   }
 
-  public LSMEngineBuilder buildRecoverManager(RecoverManager> recoverManager) {
-    lsmEngine.setRecoverManager(recoverManager);
-    return this;
-  }
-
+  /** build RecoverManager for lsmEngine */
   public LSMEngineBuilder buildRecoverManager() {
     RecoverManager> recoverManager = new RecoverManager<>(lsmEngine.getWalManager());
     lsmEngine.setRecoverManager(recoverManager);
     return this;
   }
 
+  /**
+   * build root memory node for lsmEngine
+   *
+   * @param rootMemNode root memory node
+   */
   public LSMEngineBuilder buildRootMemNode(T rootMemNode) {
     lsmEngine.setRootMemNode(rootMemNode);
     return this;
   }
 
-  public LSMEngineBuilder buildLevelProcessors(Property property) {
+  /**
+   * build level processors from ApplicationContext object
+   *
+   * @param applicationContext ApplicationContext object
+   */
+  private LSMEngineBuilder buildLevelProcessors(ApplicationContext applicationContext) {
     LevelProcessorChain insertionLevelProcessChain =
-        generateLevelProcessChain(property.getInsertionLevelProcessClass());
+        generateLevelProcessorsChain(applicationContext.getInsertionLevelProcessClass());
     LevelProcessorChain deletionLevelProcessChain =
-        generateLevelProcessChain(property.getDeletionLevelProcessClass());
+        generateLevelProcessorsChain(applicationContext.getDeletionLevelProcessClass());
     LevelProcessorChain queryLevelProcessChain =
-        generateLevelProcessChain(property.getQueryLevelProcessClass());
+        generateLevelProcessorsChain(applicationContext.getQueryLevelProcessClass());
     return buildQueryManager(queryLevelProcessChain)
         .buildInsertionManager(insertionLevelProcessChain)
         .buildDeletionManager(deletionLevelProcessChain);
   }
 
-  public LSMEngineBuilder buildLevelProcessors(String packageName) {
+  /**
+   * Scan the classes of the package and build level processors based on the class annotations
+   *
+   * @param packageName package name
+   */
+  private LSMEngineBuilder buildLevelProcessors(String packageName) {
     try {
-      Property property = PropertyGenerator.GeneratePropertyWithAnnotation(packageName);
+      ApplicationContext property =
+          ApplicationContextGenerator.GeneratePropertyWithAnnotation(packageName);
       buildLevelProcessors(property);
     } catch (Exception e) {
       logger.error(e.getMessage());
@@ -137,18 +196,32 @@ public LSMEngineBuilder buildLevelProcessors(String packageName) {
     return this;
   }
 
-  public LSMEngineBuilder buildLSMManagers(Property property, WALManager walManager) {
+  /**
+   * build all LSM managers
+   *
+   * @param applicationContext ApplicationContext object
+   * @param walManager WalManager object
+   */
+  public LSMEngineBuilder buildLSMManagers(
+      ApplicationContext applicationContext, WALManager walManager) {
     try {
-      buildWalManager(walManager).buildLevelProcessors(property).buildRecoverManager();
+      buildWalManager(walManager).buildLevelProcessors(applicationContext).buildRecoverManager();
     } catch (Exception e) {
       logger.error(e.getMessage());
     }
     return this;
   }
 
+  /**
+   * Scan the classes of the package and build all LSM managers based on the class annotations
+   *
+   * @param packageName package name
+   * @param walManager WalManager object
+   */
   public LSMEngineBuilder buildLSMManagers(String packageName, WALManager walManager) {
     try {
-      Property property = PropertyGenerator.GeneratePropertyWithAnnotation(packageName);
+      ApplicationContext property =
+          ApplicationContextGenerator.GeneratePropertyWithAnnotation(packageName);
       buildLSMManagers(property, walManager);
     } catch (Exception e) {
       logger.error(e.getMessage());
@@ -156,20 +229,34 @@ public LSMEngineBuilder buildLSMManagers(String packageName, WALManager walMa
     return this;
   }
 
+  /**
+   * Get the built lsmEngine
+   *
+   * @return LSMEngine object
+   */
   public LSMEngine build() {
     return lsmEngine;
   }
 
+  /**
+   * generate level processors Chain
+   *
+   * @param levelProcessorClassNames Save all level processor class names in hierarchical order
+   * @param  extends IRequest
+   * @param  extends RequestContext
+   * @return level Processors Chain
+   */
   private 
-      LevelProcessorChain generateLevelProcessChain(List levelProcessClassNames) {
+      LevelProcessorChain generateLevelProcessorsChain(
+          List levelProcessorClassNames) {
     LevelProcessorChain levelProcessChain = new LevelProcessorChain<>();
     try {
-      if (levelProcessClassNames.size() > 0) {
+      if (levelProcessorClassNames.size() > 0) {
         ILevelProcessor iLevelProcess =
-            levelProcessChain.nextLevel(generateLevelProcess(levelProcessClassNames.get(0)));
-        for (int i = 1; i < levelProcessClassNames.size(); i++) {
+            levelProcessChain.nextLevel(generateLevelProcessor(levelProcessorClassNames.get(0)));
+        for (int i = 1; i < levelProcessorClassNames.size(); i++) {
           iLevelProcess =
-              iLevelProcess.nextLevel(generateLevelProcess(levelProcessClassNames.get(i)));
+              iLevelProcess.nextLevel(generateLevelProcessor(levelProcessorClassNames.get(i)));
         }
       }
     } catch (Exception e) {
@@ -178,13 +265,26 @@ LevelProcessorChain generateLevelProcessChain(List levelProcess
     return levelProcessChain;
   }
 
-  private 
-      ILevelProcessor generateLevelProcess(String className)
+  /**
+   * generate level processor
+   *
+   * @param className level processor class name
+   * @param  extends IRequest
+   * @param  extends RequestContext
+   * @return level processor
+   * @throws ClassNotFoundException
+   * @throws NoSuchMethodException
+   * @throws InvocationTargetException
+   * @throws InstantiationException
+   * @throws IllegalAccessException
+   */
+  private 
+      ILevelProcessor generateLevelProcessor(String className)
           throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException,
               InstantiationException, IllegalAccessException {
     Class c = Class.forName(className);
-    ILevelProcessor result =
-        (ILevelProcessor) c.getDeclaredConstructor().newInstance();
+    ILevelProcessor result =
+        (ILevelProcessor) c.getDeclaredConstructor().newInstance();
     return result;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java
index 0b594aaedaea4..e32aad9404767 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.levelProcess;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 
 import java.util.List;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java
index 2e511b144d3b0..e46da38fee53c 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.levelProcess;
 
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext;
 
 /** indicates the deletion method of each layer of memory nodes */
 public abstract class DeleteLevelProcessor
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java
index 0b14ce8e9dce4..ca92c575174ee 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.levelProcess;
 
-import org.apache.iotdb.lsm.context.FlushRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.FlushRequestContext;
 
 /** indicates the flush method of each layer of memory nodes */
 public abstract class FlushLevelProcessor
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java
index ddf45b2489e2a..871fd180d417c 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.levelProcess;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 
 /** the processing method corresponding to each layer of memory nodes */
 public interface ILevelProcessor {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java
index c231e8f635d39..ae1240f75c087 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.levelProcess;
 
-import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext;
 
 /** indicates the insertion method of each layer of memory nodes */
 public abstract class InsertLevelProcessor
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java
index 2f035478d4f36..f1e8b9aed3913 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java
@@ -18,9 +18,10 @@
  */
 package org.apache.iotdb.lsm.levelProcess;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.request.IRequest;
 
+/** Save the level processor of each layer in hierarchical order */
 public class LevelProcessorChain {
 
   // the level process of the first layer of memory nodes
@@ -31,6 +32,13 @@ public  ILevelProcessor nextLevel(ILevelProcessor nex
     return next;
   }
 
+  /**
+   * Use the level processor of each layer to process memory nodes
+   *
+   * @param memNode memory node
+   * @param request extends IRequest
+   * @param context extends RequestContext
+   */
   public void process(T memNode, R request, C context) {
     headLevelProcess.process(memNode, request, context);
   }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java
index c7700f41193b1..1caa24844a3fb 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.levelProcess;
 
-import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext;
 
 /** indicates the query method of each layer of memory nodes */
 public abstract class QueryLevelProcessor
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
index 9dccfec9e3431..a4a3eec8486f8 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java
@@ -18,11 +18,11 @@
  */
 package org.apache.iotdb.lsm.manager;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.levelProcess.LevelProcessorChain;
 import org.apache.iotdb.lsm.request.IRequest;
 
-/** basic lsm manager */
+/** basic lsm manager implementation */
 public abstract class BasicLSMManager
     implements ILSMManager {
 
@@ -31,16 +31,11 @@ public abstract class BasicLSMManager levelProcessChain) {
-    this.levelProcessChain = levelProcessChain;
-  }
-
   /**
    * processing of the root memory node
    *
    * @param root root memory node
    * @param context request context
-   * @throws Exception
    */
   @Override
   public void process(T root, R request, C context) {
@@ -49,7 +44,12 @@ public void process(T root, R request, C context) {
     postProcess(root, request, context);
   }
 
-  public void setLevelProcessChain(LevelProcessorChain levelProcessChain) {
-    this.levelProcessChain = levelProcessChain;
+  /**
+   * set level processors chain
+   *
+   * @param levelProcessorsChain level processors chain
+   */
+  public void setLevelProcessorsChain(LevelProcessorChain levelProcessorsChain) {
+    this.levelProcessChain = levelProcessorsChain;
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
index 0519c1a40eb29..93c632ce2c777 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java
@@ -18,10 +18,10 @@
  */
 package org.apache.iotdb.lsm.manager;
 
-import org.apache.iotdb.lsm.context.DeleteRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext;
 import org.apache.iotdb.lsm.request.IDeletionRequest;
 
-/** manage deletion to MemTable */
+/** manage deletion to root memory node */
 public class DeletionManager
     extends BasicLSMManager {
 
@@ -37,7 +37,6 @@ public DeletionManager(WALManager walManager) {
    *
    * @param root root memory node
    * @param context request context
-   * @throws Exception
    */
   @Override
   public void preProcess(T root, R deletionRequest, DeleteRequestContext context) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java
index 0bffc32acbc5e..043bec8ddd4cb 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java
@@ -18,17 +18,18 @@
  */
 package org.apache.iotdb.lsm.manager;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
+import org.apache.iotdb.lsm.levelProcess.LevelProcessorChain;
+import org.apache.iotdb.lsm.request.IRequest;
 
-// used to implement lsm manager
-public interface ILSMManager {
+// Represents the manager of the lsm framework, used to handle root memory node
+public interface ILSMManager {
 
   /**
    * preprocessing of the root memory node
    *
    * @param root root memory node
    * @param context request context
-   * @throws Exception
    */
   void preProcess(T root, R request, C context);
 
@@ -37,7 +38,6 @@ public interface ILSMManager {
    *
    * @param root root memory node
    * @param context request context
-   * @throws Exception
    */
   void postProcess(T root, R request, C context);
 
@@ -48,4 +48,11 @@ public interface ILSMManager {
    * @param context request context
    */
   void process(T memNode, R request, C context);
+
+  /**
+   * set level processors chain
+   *
+   * @param levelProcessorsChain level processors chain
+   */
+  void setLevelProcessorsChain(LevelProcessorChain levelProcessorsChain);
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
index 77fb53e3fe38e..94e71d809bf70 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java
@@ -18,10 +18,10 @@
  */
 package org.apache.iotdb.lsm.manager;
 
-import org.apache.iotdb.lsm.context.InsertRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext;
 import org.apache.iotdb.lsm.request.IInsertionRequest;
 
-/** manage insertion to MemTable */
+/** manage insertion to root memory node */
 public class InsertionManager
     extends BasicLSMManager {
 
@@ -37,7 +37,6 @@ public InsertionManager(WALManager walManager) {
    *
    * @param root root memory node
    * @param context insert request context
-   * @throws Exception
    */
   @Override
   public void preProcess(T root, R insertionRequest, InsertRequestContext context) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
index 1d905e8afed39..a03d3df750b9a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java
@@ -18,10 +18,10 @@
  */
 package org.apache.iotdb.lsm.manager;
 
-import org.apache.iotdb.lsm.context.QueryRequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext;
 import org.apache.iotdb.lsm.request.IQueryRequest;
 
-/** manage query to MemTable */
+/** manage query to root memory node */
 public class QueryManager
     extends BasicLSMManager {
   @Override
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
index f3aafee5720e7..808f9b12c3511 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.manager;
 
-import org.apache.iotdb.lsm.recover.IRecoverable;
+import org.apache.iotdb.lsm.engine.IRecoverable;
 import org.apache.iotdb.lsm.request.IRequest;
 
 /** for memory structure recovery */
@@ -34,7 +34,7 @@ public RecoverManager(WALManager walManager) {
   /**
    * recover
    *
-   * @param t
+   * @param t extends IRecoverable
    */
   public void recover(T t) {
     while (true) {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
index 8d280ab0a498a..7ee7134e66a43 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java
@@ -29,7 +29,7 @@
 /** Manage wal entry writes and reads */
 public abstract class WALManager {
 
-  private final String schemaDirPath;
+  private final String walDirPath;
 
   private File walFile;
 
@@ -42,27 +42,27 @@ public abstract class WALManager {
   private boolean recover;
 
   public WALManager(String schemaDirPath) {
-    this.schemaDirPath = schemaDirPath;
+    this.walDirPath = schemaDirPath;
   }
 
   public WALManager(
-      String schemaDirPath,
+      String walDirPath,
       String walFileName,
       int walBufferSize,
       IWALRecord walRecord,
       boolean forceEachWrite)
       throws IOException {
-    this.schemaDirPath = schemaDirPath;
-    initFile(schemaDirPath, walFileName);
+    this.walDirPath = walDirPath;
+    initFile(walDirPath, walFileName);
     walWriter = new WALWriter(walFile, walBufferSize, forceEachWrite);
     walReader = new WALReader(walFile, walRecord);
     recover = false;
   }
 
-  private void initFile(String schemaDirPath, String walFileName) throws IOException {
-    File schemaDir = new File(schemaDirPath);
+  private void initFile(String walDirPath, String walFileName) throws IOException {
+    File schemaDir = new File(walDirPath);
     schemaDir.mkdirs();
-    walFile = new File(this.schemaDirPath, walFileName);
+    walFile = new File(this.walDirPath, walFileName);
     if (!walFile.exists()) {
       walFile.createNewFile();
     }
@@ -77,7 +77,7 @@ private void initFile(String schemaDirPath, String walFileName) throws IOExcepti
   public abstract void write(IRequest request);
 
   /**
-   * for recover
+   * for recover, read a wal record and generate it as a request
    *
    * @return request
    */
@@ -89,7 +89,7 @@ public void close() throws IOException {
   }
 
   public String getSchemaDirPath() {
-    return schemaDirPath;
+    return walDirPath;
   }
 
   public File getWalFile() {
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java
index 24f1e58f85bcf..9ffb218dda8c2 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java
@@ -18,12 +18,11 @@
  */
 package org.apache.iotdb.lsm.request;
 
+/** Represents a deletion request that can be processed by the lsm framework */
 public interface IDeletionRequest extends IRequest {
 
   RequestType requestType = RequestType.DELETE;
 
-  V getValue();
-
   @Override
   default RequestType getRequestType() {
     return requestType;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java
index 46d029519bf89..7bf1ae49611a1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java
@@ -18,12 +18,11 @@
  */
 package org.apache.iotdb.lsm.request;
 
+/** Represents a insertion request that can be processed by the lsm framework */
 public interface IInsertionRequest extends IRequest {
 
   RequestType requestType = RequestType.INSERT;
 
-  V getValue();
-
   @Override
   default RequestType getRequestType() {
     return requestType;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java
index d6fdf6369dee7..e2652c7ca7776 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java
@@ -18,10 +18,16 @@
  */
 package org.apache.iotdb.lsm.request;
 
+/** Represents a query request that can be processed by the lsm framework */
 public interface IQueryRequest extends IRequest {
 
   RequestType requestType = RequestType.QUERY;
 
+  @Override
+  default Object getValue() {
+    return null;
+  }
+
   @Override
   default RequestType getRequestType() {
     return requestType;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java
index ba1eca7821bd2..758ba0a779ae1 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java
@@ -18,19 +18,59 @@
  */
 package org.apache.iotdb.lsm.request;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 
 import java.util.List;
 
+/**
+ * Represents a request that can be processed by the lsm framework
+ *
+ * @param  The type of each layer key
+ * @param  The type of value
+ * @param  type of processed result
+ */
 public interface IRequest {
 
+  /**
+   * Get the key of a layer
+   *
+   * @param context request context
+   * @return the key of the layer
+   */
   K getKey(RequestContext context);
 
+  /**
+   * Get the processed result
+   *
+   * @return the processed result
+   */
   R getResult();
 
+  /**
+   * set processed result
+   *
+   * @param result processed result
+   */
   void setResult(R result);
 
+  /**
+   * get all keys
+   *
+   * @return all keys
+   */
   List getKeys();
 
+  /**
+   * get the value
+   *
+   * @return value of the request
+   */
+  V getValue();
+
+  /**
+   * get request type
+   *
+   * @return request type
+   */
   RequestType getRequestType();
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java
index 29429a074aab9..59979044ec8ef 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.lsm.request;
 
+/** Indicates the request type of request */
 public enum RequestType {
   NONE,
   INSERT,
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
index e9870d410bac0..b60f58371e5dd 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.strategy;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 import java.util.ArrayList;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java
index 89dc1d3cd9b63..2e95e5cb45357 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.strategy;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 /** access strategy for memory nodes */
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
index 46cc762092d65..9095886f589fb 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.strategy;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 import java.util.List;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
index 0ed5057671207..5f90f5ddf827d 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.strategy;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 import java.util.List;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
index 2b2f2a5744620..2daf9ecbeff0b 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java
@@ -18,7 +18,7 @@
  */
 package org.apache.iotdb.lsm.strategy;
 
-import org.apache.iotdb.lsm.context.RequestContext;
+import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor;
 
 import java.util.List;

From c7f181362b2758a85777fe2bd5c4634214d18ef9 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Tue, 25 Oct 2022 14:10:45 +0800
Subject: [PATCH 50/53] add javadoc

---
 .../tagSchemaRegion/TagSchemaRegion.java      |  31 ++++-
 .../db/engine/storagegroup/DataRegion.java    |   2 +-
 .../idtable/entry/DiskSchemaEntry.java        |   5 +
 .../metadata/schemaregion/SchemaEngine.java   |   6 -
 .../schemaregion/SchemaEngineMode.java        |   3 +-
 .../schemaregion/TagSchemaRegionLoader.java   | 113 ------------------
 .../common/schematree/ClusterSchemaTree.java  |  16 +--
 .../visitor/SchemaTreeVisitorFactory.java     | 112 -----------------
 .../db/protocol/influxdb/dto/IoTDBPoint.java  |  11 ++
 .../handler/AbstractQueryHandler.java         |  24 ++++
 .../influxdb/handler/NewQueryHandler.java     |  31 +++++
 .../influxdb/handler/QueryHandler.java        |  23 ++++
 .../influxdb/handler/QueryHandlerFactory.java |  17 +--
 .../influxdb/handler/TagQueryHandler.java     |  23 +++-
 .../meta/AbstractInfluxDBMetaManager.java     |  31 +++++
 .../influxdb/meta/IInfluxDBMetaManager.java   |  27 +++++
 .../influxdb/meta/InfluxDBMetaManager.java    |  21 ++++
 .../meta/InfluxDBMetaManagerFactory.java      |  17 +--
 .../influxdb/meta/NewInfluxDBMetaManager.java |  23 ++++
 .../influxdb/meta/TagInfluxDBMetaManager.java |   6 +-
 .../thrift/impl/NewInfluxDBServiceImpl.java   |  43 +++++++
 21 files changed, 322 insertions(+), 263 deletions(-)
 delete mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java
 delete mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
index ac0611fbca2cc..2e5ff69ab607e 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java
@@ -50,6 +50,9 @@
 import org.apache.iotdb.db.metadata.plan.schemaregion.write.IAutoCreateDeviceMNodePlan;
 import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateAlignedTimeSeriesPlan;
 import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateTimeSeriesPlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.IDeactivateTemplatePlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.IPreDeactivateTemplatePlan;
+import org.apache.iotdb.db.metadata.plan.schemaregion.write.IRollbackPreDeactivateTemplatePlan;
 import org.apache.iotdb.db.metadata.plan.schemaregion.write.ISetTemplatePlan;
 import org.apache.iotdb.db.metadata.plan.schemaregion.write.IUnsetTemplatePlan;
 import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion;
@@ -70,7 +73,7 @@
 import org.apache.iotdb.db.query.context.QueryContext;
 import org.apache.iotdb.db.query.dataset.ShowDevicesResult;
 import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult;
-import org.apache.iotdb.external.api.ISeriesNumerLimiter;
+import org.apache.iotdb.external.api.ISeriesNumerMonitor;
 import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
 import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
 import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -115,20 +118,20 @@ public class TagSchemaRegion implements ISchemaRegion {
   // manager timeSeries, and use a deviceID list manager device id -> INT32 id
   private final IDTableWithDeviceIDListImpl idTableWithDeviceIDList;
 
-  private final ISeriesNumerLimiter seriesNumerLimiter;
+  private final ISeriesNumerMonitor seriesNumerMonitor;
 
   public TagSchemaRegion(
       PartialPath storageGroup,
       SchemaRegionId schemaRegionId,
       IStorageGroupMNode storageGroupMNode,
-      ISeriesNumerLimiter seriesNumerLimiter)
+      ISeriesNumerMonitor seriesNumerMonitor)
       throws MetadataException {
     storageGroupFullPath = storageGroup.getFullPath();
     this.schemaRegionId = schemaRegionId;
     String storageGroupDirPath = config.getSchemaDir() + File.separator + storageGroupFullPath;
     schemaRegionDirPath = storageGroupDirPath + File.separator + schemaRegionId.getId();
     this.storageGroupMNode = storageGroupMNode;
-    this.seriesNumerLimiter = seriesNumerLimiter;
+    this.seriesNumerMonitor = seriesNumerMonitor;
     File schemaRegionDir = new File(schemaRegionDirPath);
     idTableWithDeviceIDList = new IDTableWithDeviceIDListImpl(schemaRegionDir);
     tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath);
@@ -935,10 +938,28 @@ public void activateSchemaTemplate(IActivateTemplateInClusterPlan plan, Template
   }
 
   @Override
-  public List getPathsUsingTemplate(int templateId) throws MetadataException {
+  public List getPathsUsingTemplate(PartialPath pathPattern, int templateId)
+      throws MetadataException {
     throw new UnsupportedOperationException("getPathsUsingTemplate");
   }
 
+  @Override
+  public int constructSchemaBlackListWithTemplate(IPreDeactivateTemplatePlan plan)
+      throws MetadataException {
+    throw new UnsupportedOperationException("constructSchemaBlackListWithTemplate");
+  }
+
+  @Override
+  public void rollbackSchemaBlackListWithTemplate(IRollbackPreDeactivateTemplatePlan plan)
+      throws MetadataException {
+    throw new UnsupportedOperationException("rollbackSchemaBlackListWithTemplate");
+  }
+
+  @Override
+  public void deactivateTemplateInBlackList(IDeactivateTemplatePlan plan) throws MetadataException {
+    throw new UnsupportedOperationException("deactivateTemplateInBlackList");
+  }
+
   @Override
   public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException {
     throw new UnsupportedOperationException("getMNodeForTrigger");
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index 6f9f384e4fead..240ea4e6867d9 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -306,7 +306,7 @@ public DataRegion(
     }
 
     // if use id table, we use id table flush time manager
-    if (config.isEnableIDTable() && !config.getSchemaEngineMode().equals("Tag")) {
+    if (config.isEnableIDTable()) {
       idTable = IDTableManager.getInstance().getIDTableDirectly(storageGroupName);
       lastFlushTimeManager = new IDTableFlushTimeManager(idTable);
     } else {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java
index f6bd30867cf18..66cb8eb3c1fd0 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java
@@ -74,6 +74,11 @@ public DiskSchemaEntry(
     this.isAligned = isAligned;
   }
 
+  /**
+   * get device path
+   *
+   * @return device path
+   */
   public String getDevicePath() {
     return seriesKey.substring(0, seriesKey.length() - measurementName.length() - 1);
   }
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
index a80fe927d49fd..c044886aad3f7 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java
@@ -323,12 +323,6 @@ private ISchemaRegion createSchemaRegionWithoutExistenceCheck(
             new RSchemaRegionLoader()
                 .loadRSchemaRegion(storageGroup, schemaRegionId, storageGroupMNode);
         break;
-      case Tag:
-        schemaRegion =
-            new TagSchemaRegionLoader()
-                .loadTagSchemaRegion(
-                    storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter);
-        break;
       default:
         throw new UnsupportedOperationException(
             String.format(
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java
index 6b3270e87612b..9147b9374c018 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java
@@ -22,6 +22,5 @@
 public enum SchemaEngineMode {
   Memory,
   Schema_File,
-  Rocksdb_based,
-  Tag
+  Rocksdb_based
 }
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java
deleted file mode 100644
index 694ffe7929d61..0000000000000
--- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.schemaregion;
-
-import org.apache.iotdb.commons.consensus.SchemaRegionId;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode;
-import org.apache.iotdb.external.api.ISeriesNumerLimiter;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.LinkedList;
-import java.util.List;
-
-public class TagSchemaRegionLoader {
-  private static final Logger LOGGER = LoggerFactory.getLogger(TagSchemaRegionLoader.class);
-  private static URLClassLoader urlClassLoader = null;
-  private static final String TAG_SCHEMA_REGION_CLASS_NAME =
-      "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaRegion";
-  private static final String LIB_PATH =
-      ".." + File.separator + "lib" + File.separator + "tag-schema-region" + File.separator;
-
-  public TagSchemaRegionLoader() {}
-
-  /**
-   * Load the jar files for TagSchemaRegion and create an instance of it. The jar files should be
-   * located in "../lib/tag-schema-region". If jar files cannot be found, the function will return
-   * null.
-   *
-   * @param storageGroup
-   * @param schemaRegionId
-   * @param node
-   * @return
-   */
-  public ISchemaRegion loadTagSchemaRegion(
-      PartialPath storageGroup,
-      SchemaRegionId schemaRegionId,
-      IStorageGroupMNode node,
-      ISeriesNumerLimiter seriesNumerLimiter) {
-    ISchemaRegion region = null;
-    LOGGER.info("Creating instance for schema-engine-tag");
-    try {
-      loadTagSchemaRegionJar();
-      Class classForTagSchemaRegion = urlClassLoader.loadClass(TAG_SCHEMA_REGION_CLASS_NAME);
-      Constructor constructor =
-          classForTagSchemaRegion.getConstructor(
-              PartialPath.class,
-              SchemaRegionId.class,
-              IStorageGroupMNode.class,
-              ISeriesNumerLimiter.class);
-
-      region =
-          (ISchemaRegion)
-              constructor.newInstance(storageGroup, schemaRegionId, node, seriesNumerLimiter);
-    } catch (ClassNotFoundException
-        | NoSuchMethodException
-        | InvocationTargetException
-        | InstantiationException
-        | IllegalAccessException
-        | MalformedURLException
-        | RuntimeException e) {
-      LOGGER.error("Cannot initialize TagSchemaRegion", e);
-      return null;
-    }
-    return region;
-  }
-
-  /**
-   * Load the jar files for TagSchemaRegion. The jar files should be located in directory
-   * "../lib/tag-schema-region". If the jar files have been loaded, it will do nothing.
-   */
-  private void loadTagSchemaRegionJar() throws MalformedURLException {
-    LOGGER.info("Loading jar for schema-engine-tag");
-    if (urlClassLoader == null) {
-      File[] jars = new File(LIB_PATH).listFiles();
-      if (jars == null) {
-        throw new RuntimeException(
-            String.format("Cannot get jars from %s", new File(LIB_PATH).getAbsolutePath()));
-      }
-      List dependentJars = new LinkedList<>();
-      for (File jar : jars) {
-        if (jar.getName().endsWith(".jar")) {
-          dependentJars.add(new URL("file:" + jar.getAbsolutePath()));
-        }
-      }
-      LOGGER.info("load jars: " + dependentJars);
-      urlClassLoader = new URLClassLoader(dependentJars.toArray(new URL[] {}));
-    }
-  }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
index 39808bbf950f1..3675f85f81667 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java
@@ -23,13 +23,13 @@
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.utils.PathUtils;
 import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.mpp.common.schematree.node.SchemaEntityNode;
 import org.apache.iotdb.db.mpp.common.schematree.node.SchemaInternalNode;
 import org.apache.iotdb.db.mpp.common.schematree.node.SchemaMeasurementNode;
 import org.apache.iotdb.db.mpp.common.schematree.node.SchemaNode;
 import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeDeviceVisitor;
 import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeMeasurementVisitor;
-import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeVisitorFactory;
 import org.apache.iotdb.tsfile.utils.Pair;
 import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
 import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -80,7 +80,12 @@ public Pair, Integer> searchMeasurementPaths(
   @Override
   public Pair, Integer> searchMeasurementPaths(PartialPath pathPattern) {
     SchemaTreeMeasurementVisitor visitor =
-        SchemaTreeVisitorFactory.getInstance().getSchemaTreeMeasurementVisitor(root, pathPattern);
+        new SchemaTreeMeasurementVisitor(
+            root,
+            pathPattern,
+            IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1,
+            0,
+            false);
     return new Pair<>(visitor.getAllResult(), visitor.getNextOffset());
   }
 
@@ -97,16 +102,13 @@ public List getAllMeasurement() {
    */
   @Override
   public List getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) {
-    SchemaTreeDeviceVisitor visitor =
-        SchemaTreeVisitorFactory.getInstance()
-            .getSchemaTreeDeviceVisitor(root, pathPattern, isPrefixMatch);
+    SchemaTreeDeviceVisitor visitor = new SchemaTreeDeviceVisitor(root, pathPattern, isPrefixMatch);
     return visitor.getAllResult();
   }
 
   @Override
   public List getMatchedDevices(PartialPath pathPattern) {
-    SchemaTreeDeviceVisitor visitor =
-        SchemaTreeVisitorFactory.getInstance().getSchemaTreeDeviceVisitor(root, pathPattern, false);
+    SchemaTreeDeviceVisitor visitor = new SchemaTreeDeviceVisitor(root, pathPattern, false);
     return visitor.getAllResult();
   }
 
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java
deleted file mode 100644
index cc669ac6f7170..0000000000000
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.mpp.common.schematree.visitor;
-
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode;
-import org.apache.iotdb.db.mpp.common.schematree.node.SchemaNode;
-
-import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_MATCH_PATTERN;
-
-/**
- * Generate corresponding SchemaTreeVisitor for different types of schema regions, such as Memory,
- * Schema_File, Rocksdb_based, and Tag
- */
-public class SchemaTreeVisitorFactory {
-
-  private SchemaTreeVisitorFactory() {}
-
-  public static SchemaTreeVisitorFactory getInstance() {
-    return SchemaTreeVisitorFactoryHolder.INSTANCE;
-  }
-
-  /**
-   * generate corresponding SchemaTreeDeviceVisitor for different types of schema regions
-   *
-   * @param schemaNode schema node
-   * @param pathPattern can be a pattern or a full path of timeseries.
-   * @param isPrefixMatch
-   * @return schemaTreeDeviceVisitor
-   */
-  public SchemaTreeDeviceVisitor getSchemaTreeDeviceVisitor(
-      SchemaNode schemaNode, PartialPath pathPattern, boolean isPrefixMatch) {
-    SchemaTreeDeviceVisitor visitor;
-    switch (SchemaEngineMode.valueOf(
-        IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) {
-      case Memory:
-      case Schema_File:
-        visitor = new SchemaTreeDeviceVisitor(schemaNode, pathPattern, isPrefixMatch);
-        break;
-      default:
-        visitor = new SchemaTreeDeviceVisitor(schemaNode, ALL_MATCH_PATTERN, isPrefixMatch);
-        break;
-    }
-    return visitor;
-  }
-
-  /**
-   * generate corresponding SchemaTreeMeasurementVisitor for different types of schema regions
-   *
-   * @param schemaNode schema node
-   * @param pathPattern can be a pattern or a full path of timeseries.
-   * @return schemaTreeMeasurementVisitor
-   */
-  public SchemaTreeMeasurementVisitor getSchemaTreeMeasurementVisitor(
-      SchemaNode schemaNode, PartialPath pathPattern) {
-    SchemaTreeMeasurementVisitor visitor;
-    switch (SchemaEngineMode.valueOf(
-        IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) {
-      case Memory:
-      case Schema_File:
-        visitor = generateSchemaTreeMeasurementVisitor(schemaNode, pathPattern);
-        break;
-      case Tag:
-        if (pathPattern.getFullPath().contains(".**")) {
-          visitor =
-              generateSchemaTreeMeasurementVisitor(
-                  schemaNode, ALL_MATCH_PATTERN.concatNode(pathPattern.getMeasurement()));
-        } else {
-          visitor = generateSchemaTreeMeasurementVisitor(schemaNode, pathPattern);
-        }
-        break;
-      default:
-        visitor = generateSchemaTreeMeasurementVisitor(schemaNode, ALL_MATCH_PATTERN);
-        break;
-    }
-    return visitor;
-  }
-
-  private SchemaTreeMeasurementVisitor generateSchemaTreeMeasurementVisitor(
-      SchemaNode schemaNode, PartialPath pathPattern) {
-    return new SchemaTreeMeasurementVisitor(
-        schemaNode,
-        pathPattern,
-        IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1,
-        0,
-        false);
-  }
-
-  private static class SchemaTreeVisitorFactoryHolder {
-    private static final SchemaTreeVisitorFactory INSTANCE = new SchemaTreeVisitorFactory();
-
-    private SchemaTreeVisitorFactoryHolder() {}
-  }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java
index 9f5aea7983839..a19f3682fa688 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java
@@ -37,6 +37,10 @@
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+/**
+ * Represent an IoTDB point, including the device path to be written, the measurement point and the
+ * corresponding value
+ */
 public class IoTDBPoint {
 
   private final String deviceId;
@@ -153,6 +157,13 @@ public InsertRowPlan convertToInsertRowPlan()
         false);
   }
 
+  /**
+   * Convert IoTDB point to InsertRecordReq
+   *
+   * @param sessionID session id
+   * @return InsertRecordReq
+   * @throws IoTDBConnectionException
+   */
   public TSInsertRecordReq convertToTSInsertRecordReq(long sessionID)
       throws IoTDBConnectionException {
     TSInsertRecordReq tsInsertRecordReq = new TSInsertRecordReq();
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java
index 10bf4dced9281..10cdef4cd04dd 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java
@@ -54,11 +54,35 @@
 import java.util.List;
 import java.util.Map;
 
+/** Used to process influxdb query requests, this abstract class defines some template methods */
 public abstract class AbstractQueryHandler {
 
+  /**
+   * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax
+   * to get the result
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param function influxdb function
+   * @param sessionid session id
+   * @return influxdb function value
+   */
   abstract InfluxFunctionValue updateByIoTDBFunc(
       String database, String measurement, InfluxFunction function, long sessionid);
 
+  /**
+   * The method that needs to be implemented is to query the result according to the query SQL
+   * supported by IoTDB
+   *
+   * @param querySql query sql
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param tagOrders tag orders
+   * @param fieldOrders field orders
+   * @param sessionId session id
+   * @return query result
+   * @throws AuthException
+   */
   abstract QueryResult queryByConditions(
       String querySql,
       String database,
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java
index 3d536df10a326..0d10431e991af 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java
@@ -31,8 +31,18 @@
 import java.util.List;
 import java.util.Map;
 
+/** query handler for NewIoTDB */
 public class NewQueryHandler extends AbstractQueryHandler {
 
+  /**
+   * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax
+   * to get the result
+   *
+   * @param path storage group path
+   * @param function influxdb function
+   * @param sessionid session id
+   * @return influxdb function value
+   */
   public final InfluxFunctionValue updateByIoTDBFunc(
       String path, InfluxFunction function, long sessionid) {
     switch (function.getFunctionName()) {
@@ -144,6 +154,16 @@ public final InfluxFunctionValue updateByIoTDBFunc(
     return function.calculateByIoTDBFunc();
   }
 
+  /**
+   * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax
+   * to get the result
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param function influxdb function
+   * @param sessionid session id
+   * @return influxdb function value
+   */
   @Override
   public InfluxFunctionValue updateByIoTDBFunc(
       String database, String measurement, InfluxFunction function, long sessionid) {
@@ -151,6 +171,17 @@ public InfluxFunctionValue updateByIoTDBFunc(
     return updateByIoTDBFunc(path, function, sessionid);
   }
 
+  /**
+   * Query the result according to the query SQL supported by IoTDB
+   *
+   * @param querySql query sql
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param tagOrders tag orders
+   * @param fieldOrders field orders
+   * @param sessionId session id
+   * @return query result
+   */
   @Override
   public QueryResult queryByConditions(
       String querySql,
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java
index 3a895bcb9a2bb..024fa011a617f 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java
@@ -51,10 +51,21 @@
 import java.util.List;
 import java.util.Map;
 
+/** query handler for IoTDB */
 public class QueryHandler extends AbstractQueryHandler {
 
   ServiceProvider serviceProvider = IoTDB.serviceProvider;
 
+  /**
+   * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax
+   * to get the result
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param function influxdb function
+   * @param sessionid session id
+   * @return influxdb function value
+   */
   @Override
   public InfluxFunctionValue updateByIoTDBFunc(
       String database, String measurement, InfluxFunction function, long sessionid) {
@@ -430,6 +441,18 @@ public InfluxFunctionValue updateByIoTDBFunc(
     return function.calculateByIoTDBFunc();
   }
 
+  /**
+   * Query the result according to the query SQL supported by IoTDB
+   *
+   * @param querySql query sql
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param tagOrders tag orders
+   * @param fieldOrders field orders
+   * @param sessionId session id
+   * @return query result
+   * @throws AuthException
+   */
   @Override
   public QueryResult queryByConditions(
       String querySql,
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java
index bfdf413a9c30b..7a6bf9bb88a3c 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java
@@ -19,22 +19,25 @@
 package org.apache.iotdb.db.protocol.influxdb.handler;
 
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode;
 import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl;
 
+/** Generate the corresponding QueryHandler object according to the configuration */
 public class QueryHandlerFactory {
+
+  /**
+   * get QueryHandler object according to the configuration
+   *
+   * @return QueryHandler object
+   */
   public static AbstractQueryHandler getInstance() {
     if (IoTDBDescriptor.getInstance()
         .getConfig()
         .getRpcImplClassName()
         .equals(ClientRPCServiceImpl.class.getName())) {
-      switch (SchemaEngineMode.valueOf(
-          IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) {
-        case Tag:
-          return new TagQueryHandler();
-        default:
-          return new NewQueryHandler();
+      if ("Tag".equals(IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) {
+        return new TagQueryHandler();
       }
+      return new NewQueryHandler();
     } else {
       return new QueryHandler();
     }
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java
index 5344bba68c90d..76a76f23219a4 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java
@@ -35,9 +35,19 @@
 import java.util.List;
 import java.util.Map;
 
-/** use in tag schema region */
+/** Query Handler for NewIoTDB When schema region is tag schema region */
 public class TagQueryHandler extends NewQueryHandler {
 
+  /**
+   * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax
+   * to get the result
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param function influxdb function
+   * @param sessionid session id
+   * @return influxdb function value
+   */
   @Override
   public InfluxFunctionValue updateByIoTDBFunc(
       String database, String measurement, InfluxFunction function, long sessionid) {
@@ -45,6 +55,17 @@ public InfluxFunctionValue updateByIoTDBFunc(
     return updateByIoTDBFunc(path, function, sessionid);
   }
 
+  /**
+   * Query the result according to the query SQL supported by IoTDB
+   *
+   * @param querySql query sql
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param tagOrders tag orders
+   * @param fieldOrders field orders
+   * @param sessionId session id
+   * @return query result
+   */
   @Override
   public QueryResult queryByConditions(
       String querySql,
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java
index 176230f39d91d..2cb1740d4c1c5 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java
@@ -24,6 +24,7 @@
 import java.util.Map;
 import java.util.Set;
 
+/** InfluxDBMetaManager used in schema region is memory or schema file */
 public abstract class AbstractInfluxDBMetaManager implements IInfluxDBMetaManager {
 
   protected static final String SELECT_TAG_INFO_SQL =
@@ -33,6 +34,14 @@ public abstract class AbstractInfluxDBMetaManager implements IInfluxDBMetaManage
   protected static Map>> database2Measurement2TagOrders =
       new HashMap<>();
 
+  /**
+   * get tag orders
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param sessionID session id
+   * @return a map of tag orders
+   */
   @Override
   public Map getTagOrders(String database, String measurement, long sessionID) {
     Map tagOrders = new HashMap<>();
@@ -47,8 +56,20 @@ public Map getTagOrders(String database, String measurement, lo
     return tagOrders;
   }
 
+  /**
+   * set storage group
+   *
+   * @param database database of influxdb
+   * @param sessionID session id
+   */
   abstract void setStorageGroup(String database, long sessionID);
 
+  /**
+   * update tag info
+   *
+   * @param tagInfoRecords tagInfoRecords
+   * @param sessionID session id
+   */
   abstract void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID);
 
   public final synchronized Map> createDatabase(
@@ -69,6 +90,16 @@ public final synchronized Map getTagOrdersWithAutoCreatingSchem
     return createDatabase(database, sessionID).computeIfAbsent(measurement, m -> new HashMap<>());
   }
 
+  /**
+   * generate time series path for insertion
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param tags influxdb tags
+   * @param fields influxdb fields
+   * @param sessionID session id
+   * @return series path
+   */
   @Override
   public final synchronized String generatePath(
       String database,
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java
index e3804ca5e76b5..3ba419b9abf41 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java
@@ -24,10 +24,29 @@
 /** used to manage influxdb metadata */
 public interface IInfluxDBMetaManager {
 
+  /** recover the influxdb metadata */
   void recover();
 
+  /**
+   * get field orders
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param sessionId session id
+   * @return a map of field orders
+   */
   Map getFieldOrders(String database, String measurement, long sessionId);
 
+  /**
+   * generate time series path for insertion
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param tags influxdb tags
+   * @param fields influxdb fields
+   * @param sessionID session id
+   * @return series path
+   */
   String generatePath(
       String database,
       String measurement,
@@ -35,5 +54,13 @@ String generatePath(
       Set fields,
       long sessionID);
 
+  /**
+   * get tag orders
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param sessionID session id
+   * @return a map of tag orders
+   */
   Map getTagOrders(String database, String measurement, long sessionID);
 }
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java
index e59d96dfbee3c..8465920353c95 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java
@@ -49,6 +49,7 @@
 import java.util.List;
 import java.util.Map;
 
+/** InfluxDBMetaManager for IoTDB When schema region is memory or schema file */
 public class InfluxDBMetaManager extends AbstractInfluxDBMetaManager {
 
   protected final Planner planner;
@@ -117,6 +118,12 @@ public void recover() {
     }
   }
 
+  /**
+   * set storage group
+   *
+   * @param database database of influxdb
+   * @param sessionID session id
+   */
   @Override
   public void setStorageGroup(String database, long sessionID) {
     try {
@@ -133,6 +140,12 @@ public void setStorageGroup(String database, long sessionID) {
     }
   }
 
+  /**
+   * update tag info
+   *
+   * @param tagInfoRecords tagInfoRecords
+   * @param sessionID session id
+   */
   @Override
   public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) {
     List plans = tagInfoRecords.convertToInsertRowPlans();
@@ -145,6 +158,14 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID)
     }
   }
 
+  /**
+   * get field orders
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param sessionID session id
+   * @return a map of field orders
+   */
   @Override
   public Map getFieldOrders(String database, String measurement, long sessionID) {
     Map fieldOrders = new HashMap<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java
index abad6b3e0ad08..de2f6c993b24c 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java
@@ -19,22 +19,25 @@
 package org.apache.iotdb.db.protocol.influxdb.meta;
 
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode;
 import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl;
 
+/** generate InfluxDBMetaManager object according to configuration */
 public class InfluxDBMetaManagerFactory {
+
+  /**
+   * generate InfluxDBMetaManager object according to configuration
+   *
+   * @return InfluxDBMetaManager object
+   */
   public static IInfluxDBMetaManager getInstance() {
     if (IoTDBDescriptor.getInstance()
         .getConfig()
         .getRpcImplClassName()
         .equals(ClientRPCServiceImpl.class.getName())) {
-      switch (SchemaEngineMode.valueOf(
-          IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) {
-        case Tag:
-          return TagInfluxDBMetaManager.getInstance();
-        default:
-          return NewInfluxDBMetaManager.getInstance();
+      if ("Tag".equals(IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) {
+        return TagInfluxDBMetaManager.getInstance();
       }
+      return NewInfluxDBMetaManager.getInstance();
     } else {
       return InfluxDBMetaManager.getInstance();
     }
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java
index 10685303c6750..2cb6a694e571b 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java
@@ -39,8 +39,10 @@
 import java.util.List;
 import java.util.Map;
 
+/** InfluxDBMetaManager for NewIoTDB When schema region is memory or schema file */
 public class NewInfluxDBMetaManager extends AbstractInfluxDBMetaManager {
 
+  // NewIoTDB uses ClientRPCServiceImpl to handle the request
   private final ClientRPCServiceImpl clientRPCService;
 
   private NewInfluxDBMetaManager() {
@@ -51,6 +53,7 @@ public static NewInfluxDBMetaManager getInstance() {
     return InfluxDBMetaManagerHolder.INSTANCE;
   }
 
+  /** recover the influxdb metadata */
   @Override
   public void recover() {
     long sessionID = 0;
@@ -99,6 +102,12 @@ public void recover() {
     }
   }
 
+  /**
+   * set storage group
+   *
+   * @param database database of influxdb
+   * @param sessionID session id
+   */
   @Override
   public void setStorageGroup(String database, long sessionID) {
     TSStatus status = clientRPCService.setStorageGroup(sessionID, "root." + database);
@@ -109,6 +118,12 @@ public void setStorageGroup(String database, long sessionID) {
     throw new InfluxDBException(status.getMessage());
   }
 
+  /**
+   * update tag info
+   *
+   * @param tagInfoRecords tagInfoRecords
+   * @param sessionID session id
+   */
   @Override
   public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) {
     try {
@@ -124,6 +139,14 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID)
     }
   }
 
+  /**
+   * get field orders
+   *
+   * @param database database of influxdb
+   * @param measurement measurement of influxdb
+   * @param sessionID session id
+   * @return a map of field orders
+   */
   @Override
   public Map getFieldOrders(String database, String measurement, long sessionID) {
     Map fieldOrders = new HashMap<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java
index 87fd7da44e7f5..29ee49356727a 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java
@@ -30,9 +30,11 @@
 import java.util.Set;
 import java.util.TreeMap;
 
-/** use in tag schema region */
+/** InfluxDBMetaManager for NewIoTDB When schema region is tag schema region */
 public class TagInfluxDBMetaManager implements IInfluxDBMetaManager {
-  private static final String STORAGE_GROUP_PATH = "root.influxdbmate";
+
+  // The storage group used to save influxdb metadata in IoTDB
+  private static final String STORAGE_GROUP_PATH = "root.influxdbmeta";
 
   private static final String TAGS_SET = "set.tags";
 
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java
index 1c40c94bc7d60..38a3a4d157c32 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java
@@ -55,6 +55,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
+/**
+ * When using NewIoTDB, use this object to handle read and write requests of the influxdb protocol
+ */
 public class NewInfluxDBServiceImpl implements IInfluxDBServiceWithHandler {
 
   private static final ClientRPCServiceImpl clientRPCService = new ClientRPCServiceImpl();
@@ -73,6 +76,13 @@ public static ClientRPCServiceImpl getClientRPCService() {
     return clientRPCService;
   }
 
+  /**
+   * open session
+   *
+   * @param req InfluxOpenSessionReq
+   * @return InfluxOpenSessionResp
+   * @throws TException
+   */
   @Override
   public InfluxOpenSessionResp openSession(InfluxOpenSessionReq req) throws TException {
     TSOpenSessionReq tsOpenSessionReq = InfluxReqAndRespUtils.convertOpenSessionReq(req);
@@ -80,6 +90,12 @@ public InfluxOpenSessionResp openSession(InfluxOpenSessionReq req) throws TExcep
     return InfluxReqAndRespUtils.convertOpenSessionResp(tsOpenSessionResp);
   }
 
+  /**
+   * close session
+   *
+   * @param req InfluxCloseSessionReq
+   * @return InfluxTSStatus
+   */
   @Override
   public InfluxTSStatus closeSession(InfluxCloseSessionReq req) {
     TSCloseSessionReq tsCloseSessionReq = InfluxReqAndRespUtils.convertCloseSessionReq(req);
@@ -87,6 +103,12 @@ public InfluxTSStatus closeSession(InfluxCloseSessionReq req) {
     return DataTypeUtils.RPCStatusToInfluxDBTSStatus(tsStatus);
   }
 
+  /**
+   * Handling insert requests
+   *
+   * @param req InfluxWritePointsReq
+   * @return InfluxTSStatus
+   */
   @Override
   public InfluxTSStatus writePoints(InfluxWritePointsReq req) {
     List tsStatusList = new ArrayList<>();
@@ -105,6 +127,12 @@ public InfluxTSStatus writePoints(InfluxWritePointsReq req) {
     return new InfluxTSStatus().setCode(executeCode).setSubStatus(tsStatusList);
   }
 
+  /**
+   * Create a database in the influxdb semantics
+   *
+   * @param req InfluxCreateDatabaseReq
+   * @return InfluxTSStatus
+   */
   @Override
   public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) {
     TSStatus tsStatus =
@@ -116,6 +144,13 @@ public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) {
     return DataTypeUtils.RPCStatusToInfluxDBTSStatus(tsStatus);
   }
 
+  /**
+   * Process query requests
+   *
+   * @param req InfluxQueryReq
+   * @return InfluxQueryResultRsp
+   * @throws TException
+   */
   @Override
   public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException {
     Operator operator = InfluxDBLogicalGenerator.generate(req.command);
@@ -123,6 +158,13 @@ public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException {
     return queryHandler.queryInfluxDB(req.database, (InfluxQueryOperator) operator, req.sessionId);
   }
 
+  /**
+   * execute sql statement
+   *
+   * @param sql sql statement
+   * @param sessionId session id
+   * @return TSExecuteStatementResp
+   */
   public static TSExecuteStatementResp executeStatement(String sql, long sessionId) {
     TSExecuteStatementReq tsExecuteStatementReq = new TSExecuteStatementReq();
     tsExecuteStatementReq.setStatement(sql);
@@ -135,6 +177,7 @@ public static TSExecuteStatementResp executeStatement(String sql, long sessionId
     return executeStatementResp;
   }
 
+  /** handle client exit, close session and resource */
   @Override
   public void handleClientExit() {
     clientRPCService.handleClientExit();

From a53f52b77e80fc1da01cde86482ebd98fb1c73f1 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Tue, 25 Oct 2022 23:06:34 +0800
Subject: [PATCH 51/53] Remove invalid exception catch code

---
 .../tagIndex/TagInvertedIndex.java            | 47 +++++++------------
 1 file changed, 18 insertions(+), 29 deletions(-)

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index 917639b55dbbb..9b9d34b4fec94 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -84,7 +84,8 @@ public TagInvertedIndex(String schemaDirPath) {
 
       // recover the lsm engine
       lsmEngine.recover();
-    } catch (Exception e) {
+    } catch (IOException e) {
+      logger.info("TagInvertedIndex initialization failed");
       logger.error(e.getMessage());
     }
   }
@@ -97,14 +98,10 @@ public TagInvertedIndex(String schemaDirPath) {
    */
   @Override
   public synchronized void addTags(Map tags, int id) {
-    try {
-      for (Map.Entry tag : tags.entrySet()) {
-        InsertionRequest insertionRequest =
-            new InsertionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
-        lsmEngine.insert(insertionRequest);
-      }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
+    for (Map.Entry tag : tags.entrySet()) {
+      InsertionRequest insertionRequest =
+          new InsertionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
+      lsmEngine.insert(insertionRequest);
     }
   }
 
@@ -116,14 +113,10 @@ public synchronized void addTags(Map tags, int id) {
    */
   @Override
   public synchronized void removeTags(Map tags, int id) {
-    try {
-      for (Map.Entry tag : tags.entrySet()) {
-        DeletionRequest deletionRequest =
-            new DeletionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
-        lsmEngine.delete(deletionRequest);
-      }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
+    for (Map.Entry tag : tags.entrySet()) {
+      DeletionRequest deletionRequest =
+          new DeletionRequest(generateKeys(tag.getKey(), tag.getValue()), id);
+      lsmEngine.delete(deletionRequest);
     }
   }
 
@@ -131,24 +124,20 @@ public synchronized void removeTags(Map tags, int id) {
    * get all matching device ids
    *
    * @param tags tags like: 
-   * @return
+   * @return ids
    */
   @Override
   public synchronized List getMatchedIDs(Map tags) {
     RoaringBitmap roaringBitmap = new RoaringBitmap();
     int i = 0;
-    try {
-      for (Map.Entry tag : tags.entrySet()) {
-        RoaringBitmap rb = getMatchedIDs(tag.getKey(), tag.getValue());
-        if (rb == null) continue;
-        else {
-          if (i == 0) roaringBitmap = rb;
-          else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb);
-          i++;
-        }
+    for (Map.Entry tag : tags.entrySet()) {
+      RoaringBitmap rb = getMatchedIDs(tag.getKey(), tag.getValue());
+      if (rb == null) continue;
+      else {
+        if (i == 0) roaringBitmap = rb;
+        else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb);
+        i++;
       }
-    } catch (Exception e) {
-      logger.error(e.getMessage());
     }
     return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList());
   }

From 79bfc6e99c7b765ce0b8565a70e79df7fd217782 Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Wed, 26 Oct 2022 13:40:36 +0800
Subject: [PATCH 52/53] add response

---
 .../tagIndex/Request/DeletionRequest.java     | 16 +----
 .../tagIndex/Request/InsertionRequest.java    | 16 +----
 .../tagIndex/Request/QueryRequest.java        | 18 +----
 .../tagIndex/TagInvertedIndex.java            |  5 +-
 .../tagIndex/query/MemChunkQuery.java         | 10 ++-
 .../tagIndex/query/MemTableQuery.java         |  2 +-
 .../tagIndex/response/QueryResponse.java      | 71 +++++++++++++++++++
 .../requestcontext/RequestContext.java        | 42 +++++++++++
 .../apache/iotdb/lsm/engine/ILSMEngine.java   | 13 ++--
 .../apache/iotdb/lsm/engine/IRecoverable.java |  3 +-
 .../apache/iotdb/lsm/engine/LSMEngine.java    | 32 +++++----
 .../iotdb/lsm/request/IDeletionRequest.java   |  2 +-
 .../iotdb/lsm/request/IInsertionRequest.java  |  2 +-
 .../iotdb/lsm/request/IQueryRequest.java      |  2 +-
 .../apache/iotdb/lsm/request/IRequest.java    | 17 +----
 .../apache/iotdb/lsm/response/IResponse.java  | 46 ++++++++++++
 16 files changed, 205 insertions(+), 92 deletions(-)
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/response/QueryResponse.java
 create mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/lsm/response/IResponse.java

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
index 632be8190e18c..1feab4c92cf11 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java
@@ -24,7 +24,7 @@
 import java.util.List;
 
 /** Represents a deletion request */
-public class DeletionRequest implements IDeletionRequest {
+public class DeletionRequest implements IDeletionRequest {
 
   // tags
   List keys;
@@ -32,14 +32,10 @@ public class DeletionRequest implements IDeletionRequest keys, int value) {
     super();
     this.keys = keys;
     this.value = value;
-    requestStatus = new RequestStatus();
   }
 
   @Override
@@ -47,16 +43,6 @@ public String getKey(RequestContext context) {
     return keys.get(context.getLevel() - 1);
   }
 
-  @Override
-  public RequestStatus getResult() {
-    return requestStatus;
-  }
-
-  @Override
-  public void setResult(RequestStatus result) {
-    requestStatus = result;
-  }
-
   @Override
   public List getKeys() {
     return keys;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
index e087917d4356a..e8c80cd8927d6 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java
@@ -24,7 +24,7 @@
 import java.util.List;
 
 /** Represents a insertion request */
-public class InsertionRequest implements IInsertionRequest {
+public class InsertionRequest implements IInsertionRequest {
 
   // tags
   List keys;
@@ -32,9 +32,6 @@ public class InsertionRequest implements IInsertionRequest keys, int value) {
     super();
     this.keys = keys;
     this.value = value;
-    requestStatus = new RequestStatus();
   }
 
   @Override
@@ -51,16 +47,6 @@ public String getKey(RequestContext context) {
     return keys.get(context.getLevel() - 1);
   }
 
-  @Override
-  public RequestStatus getResult() {
-    return requestStatus;
-  }
-
-  @Override
-  public void setResult(RequestStatus result) {
-    requestStatus = result;
-  }
-
   @Override
   public List getKeys() {
     return keys;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
index 47dd86274462d..6d4bfe4623046 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java
@@ -21,23 +21,17 @@
 import org.apache.iotdb.lsm.context.requestcontext.RequestContext;
 import org.apache.iotdb.lsm.request.IQueryRequest;
 
-import org.roaringbitmap.RoaringBitmap;
-
 import java.util.List;
 
 /** Represents a query request */
-public class QueryRequest implements IQueryRequest {
+public class QueryRequest implements IQueryRequest {
 
   // tags
   List keys;
 
-  // Query results, encapsulating ids
-  RoaringBitmap roaringBitmap;
-
   public QueryRequest(List keys) {
     super();
     this.keys = keys;
-    roaringBitmap = new RoaringBitmap();
   }
 
   @Override
@@ -45,16 +39,6 @@ public String getKey(RequestContext context) {
     return keys.get(context.getLevel() - 1);
   }
 
-  @Override
-  public RoaringBitmap getResult() {
-    return roaringBitmap;
-  }
-
-  @Override
-  public void setResult(RoaringBitmap result) {
-    roaringBitmap = result;
-  }
-
   @Override
   public List getKeys() {
     return keys;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
index 9b9d34b4fec94..a0bc369222827 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java
@@ -25,6 +25,7 @@
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.response.QueryResponse;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager;
 import org.apache.iotdb.lsm.engine.LSMEngine;
@@ -165,8 +166,8 @@ private List generateKeys(String tagKey, String tagValue) {
    */
   private RoaringBitmap getMatchedIDs(String tagKey, String tagValue) {
     QueryRequest queryRequest = new QueryRequest(generateKeys(tagKey, tagValue));
-    lsmEngine.query(queryRequest);
-    return queryRequest.getResult();
+    QueryResponse response = lsmEngine.query(queryRequest);
+    return response.getValue();
   }
 
   /**
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
index c95e5accbb34e..0f548033ed078 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java
@@ -20,6 +20,7 @@
 
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest;
 import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk;
+import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.response.QueryResponse;
 import org.apache.iotdb.lsm.annotation.QueryProcessor;
 import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext;
 import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor;
@@ -53,9 +54,14 @@ public List getChildren(
    */
   @Override
   public void query(MemChunk memNode, QueryRequest queryRequest, QueryRequestContext context) {
-    RoaringBitmap roaringBitmap = queryRequest.getResult();
+    QueryResponse response = context.getResponse();
+    if (response == null) {
+      response = new QueryResponse();
+      context.setResponse(response);
+    }
+    RoaringBitmap roaringBitmap = context.getValue();
     if (roaringBitmap == null) roaringBitmap = new RoaringBitmap();
     RoaringBitmap now = RoaringBitmap.or(roaringBitmap, memNode.getRoaringBitmap());
-    queryRequest.setResult(now);
+    context.setValue(now);
   }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
index 4fb1f42506714..e08f54cd55c49 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java
@@ -62,7 +62,7 @@ public List getChildren(
   public void query(MemTable memNode, QueryRequest queryRequest, QueryRequestContext context) {
     // if the memTable is immutable, we need to delete the id in deletionList in the query result
     if (memNode.isImmutable()) {
-      RoaringBitmap roaringBitmap = queryRequest.getResult();
+      RoaringBitmap roaringBitmap = context.getValue();
       Set deletionList = memNode.getDeletionList();
       for (Integer id : deletionList) {
         roaringBitmap.remove(id);
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/response/QueryResponse.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/response/QueryResponse.java
new file mode 100644
index 0000000000000..1d43d7ae957d2
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/response/QueryResponse.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.response;
+
+import org.apache.iotdb.lsm.response.IResponse;
+
+import org.roaringbitmap.RoaringBitmap;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** Represents a query response */
+public class QueryResponse implements IResponse {
+
+  // response value
+  private RoaringBitmap roaringBitmap;
+
+  // If an exception needs to be thrown during the processing of the request, this variable can be
+  // used to accept the exception
+  private List exceptions;
+
+  @Override
+  public RoaringBitmap getValue() {
+    return roaringBitmap;
+  }
+
+  @Override
+  public void setValue(RoaringBitmap value) {
+    this.roaringBitmap = value;
+  }
+
+  @Override
+  public List getExceptions() {
+    return exceptions;
+  }
+
+  @Override
+  public void setExceptions(List exceptions) {
+    this.exceptions = exceptions;
+  }
+
+  /**
+   * If an exception needs to be thrown during the processing of the request, this method can be
+   * used to accept the exception
+   *
+   * @param e Exception
+   */
+  @Override
+  public void addException(Exception e) {
+    if (exceptions == null) {
+      exceptions = new ArrayList<>();
+    }
+    exceptions.add(e);
+  }
+}
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java
index 607edd3698d09..626d605797414 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.lsm.context.requestcontext;
 
+import org.apache.iotdb.lsm.response.IResponse;
 import org.apache.iotdb.lsm.strategy.IAccessStrategy;
 import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy;
 
@@ -33,6 +34,9 @@ public class RequestContext {
   // the maximum level of memory nodes that can be processed
   int levelUpperBound;
 
+  // response, encapsulating the response value and exception information.
+  IResponse response;
+
   public RequestContext() {
     // preorder traversal strategy is used by default
     accessStrategy = new PreOrderAccessStrategy();
@@ -63,4 +67,42 @@ public int getLevelUpperBound() {
   public void setLevelUpperBound(int levelUpperBound) {
     this.levelUpperBound = levelUpperBound;
   }
+
+  public  R getResponse() {
+    return (R) response;
+  }
+
+  public  void setResponse(R response) {
+    this.response = response;
+  }
+
+  /**
+   * get the result of the response
+   *
+   * @param  type of the result
+   * @return response result
+   */
+  public  T getValue() {
+    return (T) getResponse().getValue();
+  }
+
+  /**
+   * set the result of the response
+   *
+   * @param value response result
+   * @param  type of the response result
+   */
+  public  void setValue(T value) {
+    response.setValue(value);
+  }
+
+  /**
+   * If an exception needs to be thrown during the processing of the request, this method can be
+   * used to accept the exception
+   *
+   * @param e Exception
+   */
+  public void thrownException(Exception e) {
+    response.addException(e);
+  }
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
index 88f473815ea41..e6c2530c47b8a 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java
@@ -22,6 +22,7 @@
 import org.apache.iotdb.lsm.request.IDeletionRequest;
 import org.apache.iotdb.lsm.request.IInsertionRequest;
 import org.apache.iotdb.lsm.request.IQueryRequest;
+import org.apache.iotdb.lsm.response.IResponse;
 
 import java.io.IOException;
 
@@ -36,18 +37,18 @@ public interface ILSMEngine extends IRecoverable {
    * @param insertionRequest Encapsulates the data to be inserted
    * @param  The type of key in the request data
    * @param  The type of value in the request data
-   * @param  return value type after insertion
+   * @param  type of response
    */
-   void insert(IInsertionRequest insertionRequest);
+   R insert(IInsertionRequest insertionRequest);
 
   /**
    * Use this ILSMEngine to query
    *
    * @param queryRequest Encapsulates query data
    * @param  The type of key in the request data
-   * @param  return value type after query
+   * @param  type of response
    */
-   void query(IQueryRequest queryRequest);
+   R query(IQueryRequest queryRequest);
 
   /**
    * Use this ILSMEngine to delete data
@@ -55,9 +56,9 @@ public interface ILSMEngine extends IRecoverable {
    * @param deletionRequest Encapsulates the data to be deleted
    * @param  The type of key in the request data
    * @param  The type of value in the request data
-   * @param  return value type after deletion
+   * @param  type of response
    */
-   void delete(IDeletionRequest deletionRequest);
+   R delete(IDeletionRequest deletionRequest);
 
   /** recover the ILSMEngine */
   void recover();
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java
index 752ae534d9122..d5fe23dda70f9 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java
@@ -29,7 +29,6 @@ public interface IRecoverable {
    * @param request insertionRequest or deletionRequest
    * @param  The type of key in the request data
    * @param  The type of value in the request data
-   * @param  return value type after recover
    */
-   void recover(IRequest request);
+   void recover(IRequest request);
 }
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
index 58c5623c2a6de..97420c173c9d6 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java
@@ -31,6 +31,7 @@
 import org.apache.iotdb.lsm.request.IInsertionRequest;
 import org.apache.iotdb.lsm.request.IQueryRequest;
 import org.apache.iotdb.lsm.request.IRequest;
+import org.apache.iotdb.lsm.response.IResponse;
 
 import java.io.IOException;
 
@@ -67,11 +68,13 @@ public LSMEngine() {}
    * @param insertionRequest Encapsulates the data to be inserted
    * @param  The type of key in the request data
    * @param  The type of value in the request data
-   * @param  return value type after insertion
+   * @param  type of response
    */
   @Override
-  public  void insert(IInsertionRequest insertionRequest) {
-    insertionManager.process(rootMemNode, insertionRequest, new InsertRequestContext());
+  public  R insert(IInsertionRequest insertionRequest) {
+    InsertRequestContext insertRequestContext = new InsertRequestContext();
+    insertionManager.process(rootMemNode, insertionRequest, insertRequestContext);
+    return insertRequestContext.getResponse();
   }
 
   /**
@@ -79,11 +82,13 @@ public  void insert(IInsertionRequest insertionRequest) {
    *
    * @param queryRequest Encapsulates query data
    * @param  The type of key in the request data
-   * @param  return value type after query
+   * @param  type of response
    */
   @Override
-  public  void query(IQueryRequest queryRequest) {
-    queryManager.process(rootMemNode, queryRequest, new QueryRequestContext());
+  public  R query(IQueryRequest queryRequest) {
+    QueryRequestContext queryRequestContext = new QueryRequestContext();
+    queryManager.process(rootMemNode, queryRequest, queryRequestContext);
+    return queryRequestContext.getResponse();
   }
 
   /**
@@ -92,11 +97,13 @@ public  void query(IQueryRequest queryRequest) {
    * @param deletionRequest Encapsulates the data to be deleted
    * @param  The type of key in the request data
    * @param  The type of value in the request data
-   * @param  return value type after deletion
+   * @param  type of response
    */
   @Override
-  public  void delete(IDeletionRequest deletionRequest) {
-    deletionManager.process(rootMemNode, deletionRequest, new DeleteRequestContext());
+  public  R delete(IDeletionRequest deletionRequest) {
+    DeleteRequestContext deleteRequestContext = new DeleteRequestContext();
+    deletionManager.process(rootMemNode, deletionRequest, deleteRequestContext);
+    return deleteRequestContext.getResponse();
   }
 
   /** recover the LSMEngine */
@@ -122,16 +129,15 @@ public void clear() throws IOException {
    * @param request insertionRequest or deletionRequest
    * @param  The type of key in the request data
    * @param  The type of value in the request data
-   * @param  return value type after deletion
    */
   @Override
-  public  void recover(IRequest request) {
+  public  void recover(IRequest request) {
     switch (request.getRequestType()) {
       case INSERT:
-        insert((IInsertionRequest) request);
+        insert((IInsertionRequest) request);
         break;
       case DELETE:
-        delete((IDeletionRequest) request);
+        delete((IDeletionRequest) request);
         break;
       default:
         break;
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java
index 9ffb218dda8c2..e457fdf5e02f2 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.request;
 
 /** Represents a deletion request that can be processed by the lsm framework */
-public interface IDeletionRequest extends IRequest {
+public interface IDeletionRequest extends IRequest {
 
   RequestType requestType = RequestType.DELETE;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java
index 7bf1ae49611a1..b6c7e4ff983ba 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.request;
 
 /** Represents a insertion request that can be processed by the lsm framework */
-public interface IInsertionRequest extends IRequest {
+public interface IInsertionRequest extends IRequest {
 
   RequestType requestType = RequestType.INSERT;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java
index e2652c7ca7776..92ee326a353ea 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java
@@ -19,7 +19,7 @@
 package org.apache.iotdb.lsm.request;
 
 /** Represents a query request that can be processed by the lsm framework */
-public interface IQueryRequest extends IRequest {
+public interface IQueryRequest extends IRequest {
 
   RequestType requestType = RequestType.QUERY;
 
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java
index 758ba0a779ae1..a2a810067c3db 100644
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java
@@ -27,9 +27,8 @@
  *
  * @param  The type of each layer key
  * @param  The type of value
- * @param  type of processed result
  */
-public interface IRequest {
+public interface IRequest {
 
   /**
    * Get the key of a layer
@@ -39,20 +38,6 @@ public interface IRequest {
    */
   K getKey(RequestContext context);
 
-  /**
-   * Get the processed result
-   *
-   * @return the processed result
-   */
-  R getResult();
-
-  /**
-   * set processed result
-   *
-   * @param result processed result
-   */
-  void setResult(R result);
-
   /**
    * get all keys
    *
diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/response/IResponse.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/response/IResponse.java
new file mode 100644
index 0000000000000..420685577edb3
--- /dev/null
+++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/response/IResponse.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.lsm.response;
+
+import java.util.List;
+
+/**
+ * Indicates the response after the lsm framework processes the request, encapsulating the response
+ * value and exception information.
+ *
+ * @param  type of the response result
+ */
+public interface IResponse {
+
+  T getValue();
+
+  void setValue(T value);
+
+  List getExceptions();
+
+  void setExceptions(List exceptions);
+
+  /**
+   * If an exception needs to be thrown during the processing of the request, this method can be
+   * used to accept the exception
+   *
+   * @param e Exception
+   */
+  void addException(Exception e);
+}

From 01c424e98913bd9f9187a5f35fdfaa217ecab86d Mon Sep 17 00:00:00 2001
From: KeePromMise 
Date: Wed, 26 Oct 2022 13:47:19 +0800
Subject: [PATCH 53/53] Remove unnecessary response

---
 .../tagIndex/Request/RequestStatus.java       | 41 -------------------
 1 file changed, 41 deletions(-)
 delete mode 100644 schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java

diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java
deleted file mode 100644
index fc81aed24c603..0000000000000
--- a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/RequestStatus.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request;
-
-/** Indicates the status of the request, whether it is successful after the request is processed */
-public class RequestStatus {
-
-  Boolean success;
-
-  public RequestStatus() {
-    success = true;
-  }
-
-  public RequestStatus(Boolean success) {
-    this.success = success;
-  }
-
-  public Boolean getSuccess() {
-    return success;
-  }
-
-  public void setSuccess(Boolean success) {
-    this.success = success;
-  }
-}