diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index 31c9b49bc380..192d0ba3cd4d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -976,11 +976,28 @@ public static void executeNonQueriesWithRetry( } } - public static void executeNonQuery(BaseEnv env, String sql, Connection defaultConnection) { + public static void executeNonQuery(final BaseEnv env, final String sql) { + executeNonQuery(env, sql, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, null); + } + + public static void executeNonQuery( + final BaseEnv env, final String sql, final Connection defaultConnection) { executeNonQuery( env, sql, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, defaultConnection); } + public static void executeNonQuery( + final String dataBaseName, final String sqlDialect, final BaseEnv env, final String sql) { + executeNonQuery( + env, + sql, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + dataBaseName, + sqlDialect, + null); + } + public static void executeNonQuery( String dataBaseName, String sqlDialect, @@ -1020,6 +1037,17 @@ public static void executeNonQuery( defaultConnection); } + public static void executeNonQueries(BaseEnv env, List sqlList) { + executeNonQueries( + env, + sqlList, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + null, + TREE_SQL_DIALECT, + null); + } + public static void executeNonQueries( BaseEnv env, List sqlList, Connection defaultConnection) { executeNonQueries( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java index a11cc5cc80c6..3fd03bebd758 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java @@ -75,22 +75,22 @@ public void testLifeCycleWithHistoryEnabled() throws Exception { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("user", "root"); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -139,29 +139,29 @@ public void testLifeCycleWithHistoryDisabled() throws Exception { TestUtils.executeNonQueries(senderEnv, Collections.singletonList("flush"), null); Thread.sleep(10000); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("user", "root"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("user", "root"); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("source.inclusion.exclusion", ""); - extractorAttributes.put("extractor.history.enable", "false"); + sourceAttributes.put("source.history.enable", "false"); // start-time and end-time should not work - extractorAttributes.put("extractor.history.start-time", "0"); - extractorAttributes.put("extractor.history.end-time", "50"); + sourceAttributes.put("source.history.start-time", "0"); + sourceAttributes.put("source.history.end-time", "50"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -196,23 +196,23 @@ public void testLifeCycleLogMode() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("extractor.mode", "forced-log"); - extractorAttributes.put("user", "root"); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("source.mode", "forced-log"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -252,23 +252,23 @@ public void testLifeCycleFileMode() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("mode.streaming", "false"); - extractorAttributes.put("user", "root"); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("mode.streaming", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -306,23 +306,23 @@ public void testLifeCycleHybridMode() throws Exception { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("extractor.mode", "hybrid"); - extractorAttributes.put("user", "root"); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("source.mode", "hybrid"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -358,22 +358,22 @@ public void testLifeCycleWithClusterRestart() throws Exception { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("user", "root"); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -421,22 +421,22 @@ public void testReceiverRestartWhenTransferring() throws Exception { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("user", "root"); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -500,22 +500,22 @@ public void testReceiverAlreadyHaveTimeSeries() throws Exception { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("user", "root"); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -559,24 +559,24 @@ public void testDoubleLiving() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); // Add this property to avoid to make self cycle. - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("forwarding-pipe-requests", "false"); - extractorAttributes.put("user", "root"); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -592,25 +592,25 @@ public void testDoubleLiving() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); // Add this property to avoid to make self cycle. - extractorAttributes.put("capture.table", "true"); - extractorAttributes.put("capture.tree", "true"); - extractorAttributes.put("forwarding-pipe-requests", "false"); - extractorAttributes.put("user", "root"); + sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("capture.tree", "true"); + sourceAttributes.put("forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", senderIp); + sinkAttributes.put("sink.port", Integer.toString(senderPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -643,10 +643,10 @@ public void testPermission() { assertTableNonQueryTestFail( senderEnv, "create pipe testPipe\n" - + "with connector (\n" - + " 'connector'='iotdb-thrift-connector',\n" - + " 'connector.ip'='127.0.0.1',\n" - + " 'connector.port'='6668'\n" + + "with sink (\n" + + " 'sink'='iotdb-thrift-sink',\n" + + " 'sink.ip'='127.0.0.1',\n" + + " 'sink.port'='6668'\n" + ")", "803: Access Denied: No permissions for this operation, please add privilege SYSTEM", "test", diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java index 958f9d409351..f233596fc60c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java @@ -184,11 +184,11 @@ public void testSourcePermission() { // Grant some privilege TestUtils.executeNonQuery( - "test", BaseEnv.TABLE_SQL_DIALECT, senderEnv, "grant INSERT on any to user thulab", null); + "test", BaseEnv.TABLE_SQL_DIALECT, senderEnv, "grant INSERT on any to user thulab"); TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test1"); - // Shall not be transferred + // Shall be transferred TestUtils.assertDataEventuallyOnEnv( receiverEnv, "show tables from test1", diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java index 03e3efdac916..ffb2fe902cf6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java @@ -215,28 +215,28 @@ private void testTableConfigIdempotent(final List beforeSqlList, final S try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("extractor.capture.tree", "false"); - extractorAttributes.put("user", "root"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.capture.table", "true"); + sourceAttributes.put("source.capture.tree", "false"); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java index d828a5054e3f..0a927cab2985 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java @@ -69,22 +69,23 @@ public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { Arrays.asList("insert into root.vehicle.d0(time, s1) values (0, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.realtime.mode", "log"); + sourceAttributes.put("source.realtime.mode", "log"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.realtime-first", "false"); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.realtime-first", "false"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -133,25 +134,26 @@ private void testSinkFormat(final String format) throws Exception { Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.realtime.mode", "forced-log"); + sourceAttributes.put("source.realtime.mode", "forced-log"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.format", format); - connectorAttributes.put("connector.realtime-first", "false"); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.format", format); + sinkAttributes.put("sink.realtime-first", "false"); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client .createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)) .getCode()); @@ -178,8 +180,8 @@ private void testSinkFormat(final String format) throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client .createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)) .getCode()); @@ -211,24 +213,25 @@ public void testLegacyConnector() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.realtime.mode", "log"); + sourceAttributes.put("source.realtime.mode", "log"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("sink", "iotdb-legacy-pipe-sink"); - connectorAttributes.put("sink.batch.enable", "false"); - connectorAttributes.put("sink.ip", receiverIp); - connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-legacy-pipe-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); // This version does not matter since it's no longer checked by the legacy receiver - connectorAttributes.put("sink.version", "1.3"); + sinkAttributes.put("sink.version", "1.3"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -257,6 +260,7 @@ public void testReceiverAutoCreateByLog() throws Exception { new HashMap() { { put("source.realtime.mode", "forced-log"); + put("user", "root"); } }); } @@ -267,6 +271,7 @@ public void testReceiverAutoCreateByFile() throws Exception { new HashMap() { { put("source.realtime.mode", "batch"); + put("user", "root"); } }); } @@ -278,12 +283,12 @@ public void testReceiverAutoCreateWithPattern() throws Exception { { put("source.realtime.mode", "batch"); put("source.path", "root.ln.wf01.wt0*.*"); + put("user", "root"); } }); } - private void testReceiverAutoCreate(final Map extractorAttributes) - throws Exception { + private void testReceiverAutoCreate(final Map sourceAttributes) throws Exception { final Consumer handleFailure = o -> { TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); @@ -298,17 +303,17 @@ private void testReceiverAutoCreate(final Map extractorAttribute try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("sink", "iotdb-thrift-sink"); - connectorAttributes.put("sink.batch.enable", "false"); - connectorAttributes.put("sink.ip", receiverIp); - connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -389,24 +394,25 @@ private void testReceiverLoadTsFile(final String loadTsFileStrategy) throws Exce Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.realtime.mode", "forced-log"); + sourceAttributes.put("source.realtime.mode", "forced-log"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("sink", "iotdb-thrift-sink"); - connectorAttributes.put("sink.batch.enable", "false"); - connectorAttributes.put("sink.ip", receiverIp); - connectorAttributes.put("sink.port", Integer.toString(receiverPort)); - connectorAttributes.put("sink.load-tsfile-strategy", loadTsFileStrategy); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.load-tsfile-strategy", loadTsFileStrategy); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client .createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)) .getCode()); @@ -433,8 +439,8 @@ private void testReceiverLoadTsFile(final String loadTsFileStrategy) throws Exce TSStatusCode.SUCCESS_STATUS.getStatusCode(), client .createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)) .getCode()); @@ -483,25 +489,26 @@ private void testLoadTsFileWithoutVerify(final String loadTsFileStrategy) throws Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.realtime.mode", "batch"); + sourceAttributes.put("source.realtime.mode", "batch"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("sink", "iotdb-thrift-sink"); - connectorAttributes.put("sink.batch.enable", "false"); - connectorAttributes.put("sink.ip", receiverIp); - connectorAttributes.put("sink.port", Integer.toString(receiverPort)); - connectorAttributes.put("sink.load-tsfile-strategy", loadTsFileStrategy); - connectorAttributes.put("sink.tsfile.validation", "false"); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.load-tsfile-strategy", loadTsFileStrategy); + sinkAttributes.put("sink.tsfile.validation", "false"); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client .createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)) .getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java index b105b701ca10..b4519ca95009 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java @@ -78,19 +78,21 @@ public void testLifeCycleWithHistoryEnabled() throws Exception { Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -147,27 +149,28 @@ public void testLifeCycleWithHistoryDisabled() throws Exception { Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); - extractorAttributes.put("extractor.history.enable", "false"); + sourceAttributes.put("source.history.enable", "false"); // start-time and end-time should not work - extractorAttributes.put("extractor.history.start-time", "0001.01.01T00:00:00"); - extractorAttributes.put("extractor.history.end-time", "2100.01.01T00:00:00"); + sourceAttributes.put("source.history.start-time", "0001.01.01T00:00:00"); + sourceAttributes.put("source.history.end-time", "2100.01.01T00:00:00"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -274,21 +277,22 @@ public void testLifeCycleLogMode() throws Exception { Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.mode", "log"); + sourceAttributes.put("source.mode", "log"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -338,21 +342,22 @@ public void testLifeCycleFileMode() throws Exception { Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.mode", "file"); + sourceAttributes.put("source.mode", "file"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -402,21 +407,22 @@ public void testLifeCycleHybridMode() throws Exception { Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.mode", "hybrid"); + sourceAttributes.put("source.mode", "hybrid"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -467,19 +473,21 @@ public void testLifeCycleWithClusterRestart() throws Exception { Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -533,19 +541,21 @@ public void testReceiverRestartWhenTransferring() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -609,19 +619,21 @@ public void testReceiverAlreadyHaveTimeSeries() throws Exception { TestUtils.executeNonQuery( receiverEnv, "insert into root.db.d1(time, s1) values (1, 1)", null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -675,22 +687,23 @@ public void testDoubleLiving() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); // Add this property to avoid to make self cycle. - extractorAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -719,22 +732,23 @@ public void testDoubleLiving() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); // Add this property to avoid to make self cycle. - extractorAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", senderIp); + sinkAttributes.put("sink.port", Integer.toString(senderPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -805,10 +819,10 @@ public void testPermission() { assertNonQueryTestFail( senderEnv, "create pipe testPipe\n" - + "with connector (\n" - + " 'connector'='iotdb-thrift-connector',\n" - + " 'connector.ip'='127.0.0.1',\n" - + " 'connector.port'='6668'\n" + + "with sink (\n" + + " 'sink'='iotdb-thrift-sink',\n" + + " 'sink.ip'='127.0.0.1',\n" + + " 'sink.port'='6668'\n" + ")", "803: No permissions for this operation, please add privilege SYSTEM", "test", @@ -861,10 +875,7 @@ public void testPermission() { executeNonQueryWithRetry( senderEnv, - "create pipe testPipe\n" - + "with connector (\n" - + " 'connector'='write-back-connector'\n" - + ")", + "create pipe testPipe\n" + "with sink (\n" + " 'sink'='write-back-sink'\n" + ")", "test", "test123123456"); executeQueryWithRetry(senderEnv, "show pipes", "test", "test123123456"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java index 0ffdac38ca71..dfa7957a5746 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java @@ -101,25 +101,26 @@ public void testTumblingTimeSamplingProcessor() throws Exception { "insert into root.vehicle.d0(time, s1) values (0, 1)", "delete from root.**"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.realtime.mode", "log"); + sourceAttributes.put("source.realtime.mode", "log"); + sourceAttributes.put("user", "root"); processorAttributes.put("processor", "tumbling-time-sampling-processor"); processorAttributes.put("processor.tumbling-time.interval-seconds", "20"); processorAttributes.put("processor.down-sampling.split-file", "true"); - connectorAttributes.put("sink", "iotdb-thrift-sink"); - connectorAttributes.put("sink.batch.enable", "false"); - connectorAttributes.put("sink.ip", receiverIp); - connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java index 0d6091c51eca..b5e4127e9465 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java @@ -200,19 +200,21 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { TestUtils.executeNonQuery(senderEnv, "insert into root.db.d1(time, s1) values (1, 1)", null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -238,19 +240,21 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { TestUtils.executeNonQuery( receiverEnv, "insert into root.db.d1(time, s1) values (2, 2)", null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", senderIp); + sinkAttributes.put("sink.port", Integer.toString(senderPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -276,19 +280,21 @@ private void doTest() throws Exception { TestUtils.executeNonQuery(senderEnv, "insert into root.db.d1(time, s1) values (1, 1)", null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -344,7 +350,7 @@ public void testAirGapConnectorUseNodeUrls() throws Exception { doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); } - private void doTestUseNodeUrls(String connectorName) throws Exception { + private void doTestUseNodeUrls(String sinkName) throws Exception { senderEnv .getConfig() .getCommonConfig() @@ -378,7 +384,7 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { final StringBuilder nodeUrlsBuilder = new StringBuilder(); for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { - if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) { + if (sinkName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) { // Use default port for convenience nodeUrlsBuilder .append(wrapper.getIp()) @@ -403,24 +409,25 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { "delete from root.db.d1.s1 where time > 2"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", connectorName); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.node-urls", nodeUrlsBuilder.toString()); + sinkAttributes.put("sink", sinkName); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.node-urls", nodeUrlsBuilder.toString()); - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.mods.enable", "true"); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.mods.enable", "true"); + sourceAttributes.put("user", "root"); // Test forced-log mode, in open releases this might be "file" - extractorAttributes.put("source.realtime.mode", "forced-log"); + sourceAttributes.put("source.realtime.mode", "forced-log"); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -459,13 +466,13 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { receiverEnv, "count databases", "count,", Collections.singleton("3,")); // Test file mode - extractorAttributes.put("source.inclusion", "data"); - extractorAttributes.replace("source.realtime.mode", "file"); + sourceAttributes.put("source.inclusion", "data"); + sourceAttributes.replace("source.realtime.mode", "file"); status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSinkParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSinkParallelIT.java index 9cd61d68fbf1..c21e6456ef14 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSinkParallelIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSinkParallelIT.java @@ -61,20 +61,22 @@ public void testIoTConnectorParallel() throws Exception { final Set expectedResSet = new HashSet<>(); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.parallel.tasks", "3"); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.parallel.tasks", "3"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java index be4b69065331..a45115f5a494 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java @@ -104,21 +104,21 @@ public void testExtractorValidParameter() throws Exception { final int receiverPort = receiverDataNode.getPort(); // ---------------------------------------------------------- // - // Scenario 1: when 'extractor.history.enable' is set to true // + // Scenario 1: when 'source.history.enable' is set to true // // ---------------------------------------------------------- // - // Scenario 1.1: test when 'extractor.history.start-time' and 'extractor.history.end-time' are + // Scenario 1.1: test when 'source.history.start-time' and 'source.history.end-time' are // not set final String p1_1 = String.format( "create pipe p1_1" - + " with extractor (" - + "'extractor.history.enable'='true')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with source (" + + "'source.history.enable'='true')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -127,18 +127,18 @@ public void testExtractorValidParameter() throws Exception { fail(e.getMessage()); } - // Scenario 1.2: test when only 'extractor.history.start-time' is set + // Scenario 1.2: test when only 'source.history.start-time' is set final String p1_2 = String.format( "create pipe p1_2" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -147,18 +147,18 @@ public void testExtractorValidParameter() throws Exception { fail(e.getMessage()); } - // Scenario 1.3: test when only 'extractor.history.end-time' is set + // Scenario 1.3: test when only 'source.history.end-time' is set final String p1_3 = String.format( "create pipe p1_3" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -167,19 +167,19 @@ public void testExtractorValidParameter() throws Exception { fail(e.getMessage()); } - // Scenario 1.4: test when 'extractor.history.start-time' equals 'extractor.history.end-time' + // Scenario 1.4: test when 'source.history.start-time' equals 'source.history.end-time' final String p1_4 = String.format( "create pipe p1_4" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'='2000.01.01T08:00:00'," - + "'extractor.history.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'='2000.01.01T08:00:00'," + + "'source.history.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -188,19 +188,19 @@ public void testExtractorValidParameter() throws Exception { fail(e.getMessage()); } - // Scenario 1.5: test when 'extractor.history.end-time' is future time + // Scenario 1.5: test when 'source.history.end-time' is future time final String p1_5 = String.format( "create pipe p1_5" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'='2000.01.01T08:00:00'," - + "'extractor.history.end-time'='2100.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'='2000.01.01T08:00:00'," + + "'source.history.end-time'='2100.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -221,11 +221,11 @@ public void testExtractorValidParameter() throws Exception { "create pipe p2_1" + " with source (" + "'source.start-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -240,11 +240,11 @@ public void testExtractorValidParameter() throws Exception { "create pipe p2_2" + " with source (" + "'source.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -260,11 +260,11 @@ public void testExtractorValidParameter() throws Exception { + " with source (" + "'source.start-time'='2000.01.01T08:00:00'," + "'source.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -280,11 +280,11 @@ public void testExtractorValidParameter() throws Exception { + " with source (" + "'source.start-time'='2000.01.01T08:00:00'," + "'source.end-time'='2100.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -306,11 +306,11 @@ public void testExtractorValidParameter() throws Exception { + " with source (" + "'source.start-time'='1000'," + "'source.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -329,18 +329,18 @@ public void testExtractorInvalidParameter() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - // Scenario 1: invalid 'extractor.history.start-time' + // Scenario 1: invalid 'source.history.start-time' final String formatString = String.format( "create pipe p1" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'=%s)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'=%s)" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", "%s", receiverIp, receiverPort); final List invalidStartTimes = @@ -355,19 +355,19 @@ public void testExtractorInvalidParameter() throws Exception { } assertPipeCount(0); - // Scenario 2: can not set 'extractor.history.enable' and 'extractor.realtime.enable' both to + // Scenario 2: can not set 'source.history.enable' and 'source.realtime.enable' both to // false final String p2 = String.format( "create pipe p2" - + " with extractor (" - + "'extractor.history.enable'='false'," - + "'extractor.realtime.enable'='false')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with source (" + + "'source.history.enable'='false'," + + "'source.realtime.enable'='false')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -377,20 +377,20 @@ public void testExtractorInvalidParameter() throws Exception { } assertPipeCount(0); - // Scenario 3: test when 'extractor.history.start-time' is greater than - // 'extractor.history.end-time' + // Scenario 3: test when 'source.history.start-time' is greater than + // 'source.history.end-time' final String p3 = String.format( "create pipe p3" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.history.start-time'='2001.01.01T08:00:00'," - + "'extractor.history.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with source (" + + "'source.history.enable'='true'," + + "'source.history.start-time'='2001.01.01T08:00:00'," + + "'source.history.end-time'='2000.01.01T08:00:00')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -407,11 +407,11 @@ public void testExtractorInvalidParameter() throws Exception { + " with source (" + "'source.start-time'='2001.01.01T08:00:00'," + "'source.end-time'='2000.01.01T08:00:00')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -450,17 +450,18 @@ public void testExtractorPatternMatch() throws Exception { + "aligned values (now(), 0.5, 1, 1.5, 2, \"text1\", true)"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.pattern", null); - extractorAttributes.put("extractor.inclusion", "data.insert"); + sourceAttributes.put("source.pattern", null); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final List patterns = Arrays.asList( @@ -482,11 +483,11 @@ public void testExtractorPatternMatch() throws Exception { Arrays.asList(0, 1, 2, 3, 4, 5, 6, 11, 16, 16, 18, 20); for (int i = 0; i < patterns.size(); ++i) { - extractorAttributes.replace("extractor.pattern", patterns.get(i)); + sourceAttributes.replace("source.pattern", patterns.get(i)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p" + i, connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p" + i, sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -534,22 +535,23 @@ public void testMatchingMultipleDatabases() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.pattern", "root.db1"); - extractorAttributes.put("extractor.inclusion", "data.insert"); + sourceAttributes.put("source.pattern", "root.db1"); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -564,11 +566,11 @@ public void testMatchingMultipleDatabases() throws Exception { "flush"), null); - extractorAttributes.replace("extractor.pattern", "root.db2"); + sourceAttributes.replace("source.pattern", "root.db2"); status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -588,11 +590,11 @@ public void testMatchingMultipleDatabases() throws Exception { "flush"), null); - extractorAttributes.remove("extractor.pattern"); // no pattern, will match all databases + sourceAttributes.remove("source.pattern"); // no pattern, will match all databases status = client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p3", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -625,47 +627,48 @@ public void testHistoryAndRealtime() throws Exception { "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - extractorAttributes.put("extractor.inclusion", "data.insert"); - extractorAttributes.put("extractor.pattern", "root.db.d2"); - extractorAttributes.put("extractor.history.enable", "false"); - extractorAttributes.put("extractor.realtime.enable", "true"); + final Map sinkAttributes = new HashMap<>(); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("source.pattern", "root.db.d2"); + sourceAttributes.put("source.history.enable", "false"); + sourceAttributes.put("source.realtime.enable", "true"); + sourceAttributes.put("user", "root"); TSStatus status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p2").getCode()); - extractorAttributes.replace("extractor.pattern", "root.db.d3"); - extractorAttributes.replace("extractor.history.enable", "true"); - extractorAttributes.replace("extractor.realtime.enable", "false"); + sourceAttributes.replace("source.pattern", "root.db.d3"); + sourceAttributes.replace("source.history.enable", "true"); + sourceAttributes.replace("source.realtime.enable", "false"); status = client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p3", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p3").getCode()); - extractorAttributes.replace("extractor.pattern", "root.db.d4"); - extractorAttributes.replace("extractor.history.enable", "true"); - extractorAttributes.replace("extractor.realtime.enable", "true"); + sourceAttributes.replace("source.pattern", "root.db.d4"); + sourceAttributes.replace("source.history.enable", "true"); + sourceAttributes.replace("source.realtime.enable", "true"); status = client.createPipe( - new TCreatePipeReq("p4", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p4", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -712,26 +715,27 @@ public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws E "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.pattern", "root.db.d1"); - extractorAttributes.put("extractor.inclusion", "data.insert"); - extractorAttributes.put("extractor.history.enable", "true"); + sourceAttributes.put("source.pattern", "root.db.d1"); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("source.history.enable", "true"); // 1970-01-01T08:00:02+08:00 - extractorAttributes.put("extractor.history.start-time", "2000"); - extractorAttributes.put("extractor.history.end-time", "1970-01-01T08:00:04+08:00"); + sourceAttributes.put("source.history.start-time", "2000"); + sourceAttributes.put("source.history.end-time", "1970-01-01T08:00:04+08:00"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -743,11 +747,11 @@ public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws E "count(root.db.d1.at1),", Collections.singleton("3,")); - extractorAttributes.remove("extractor.pattern"); + sourceAttributes.remove("source.pattern"); status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -781,23 +785,24 @@ public void testExtractorTimeRangeMatch() throws Exception { "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); - extractorAttributes.put("source.inclusion", "data"); - extractorAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); - extractorAttributes.put("source.end-time", "1970-01-01T08:00:04+08:00"); + sourceAttributes.put("source.inclusion", "data"); + sourceAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); + sourceAttributes.put("source.end-time", "1970-01-01T08:00:04+08:00"); + sourceAttributes.put("user", "root"); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -858,25 +863,26 @@ public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Ex "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.pattern", "root.db.d1"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); + sourceAttributes.put("source.pattern", "root.db.d1"); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); // 1970-01-01T08:00:04+08:00 - extractorAttributes.put("source.end-time", "4000"); + sourceAttributes.put("source.end-time", "4000"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -888,11 +894,11 @@ public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Ex "count(root.db.d1.at1),", Collections.singleton("3,")); - extractorAttributes.remove("source.pattern"); + sourceAttributes.remove("source.pattern"); status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -933,25 +939,26 @@ public void testHistoryLooseRange() throws Exception { "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.path", "root.db.d1.at1"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.history.start-time", "1500"); - extractorAttributes.put("source.history.end-time", "2500"); - extractorAttributes.put("source.history.loose-range", "time, path"); + sourceAttributes.put("source.path", "root.db.d1.at1"); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("source.history.start-time", "1500"); + sourceAttributes.put("source.history.end-time", "2500"); + sourceAttributes.put("source.history.loose-range", "time, path"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -971,28 +978,29 @@ public void testRealtimeLooseRange() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.path", "root.db.d1.at1"); - extractorAttributes.put("source.inclusion", "data.insert"); - extractorAttributes.put("source.realtime.loose-range", "time, path"); - extractorAttributes.put("source.start-time", "2000"); - extractorAttributes.put("source.end-time", "10000"); - extractorAttributes.put("source.realtime.mode", "batch"); + sourceAttributes.put("source.path", "root.db.d1.at1"); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("source.realtime.loose-range", "time, path"); + sourceAttributes.put("source.start-time", "2000"); + sourceAttributes.put("source.end-time", "10000"); + sourceAttributes.put("source.realtime.mode", "batch"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java index c89fef3a3f61..ce197b943560 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java @@ -62,33 +62,35 @@ public void testPipeSwitchStatus() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); status = client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p3", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -149,19 +151,21 @@ public void testPipeIllegallySwitchStatus() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -173,8 +177,8 @@ public void testPipeIllegallySwitchStatus() throws Exception { status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), status.getCode()); @@ -222,19 +226,21 @@ public void testDropPipeAndCreateAgain() throws Exception { (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { TestUtils.executeNonQuery(senderEnv, "insert into root.db.d1(time, s1) values (1, 1)", null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( @@ -260,8 +266,8 @@ public void testDropPipeAndCreateAgain() throws Exception { status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); showPipeResult = @@ -280,19 +286,21 @@ public void testWrongPipeName() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java index e925e9e44284..dc36b2a19ad3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java @@ -77,11 +77,11 @@ public void testValidPipeName() throws Exception { statement.execute( String.format( "create pipe %s" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", pipeName, receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -127,14 +127,14 @@ public void testRevertParameterOrder() { statement.execute( String.format( "create pipe p1" - + " with extractor (" - + "'extractor.realtime.mode'='hybrid'," - + "'extractor.history.enable'='false') " - + " with connector (" - + "'connector.batch.enable'='false', " - + "'connector.port'='%s'," - + "'connector.ip'='%s'," - + "'connector'='iotdb-thrift-connector')", + + " with source (" + + "'source.realtime.mode'='hybrid'," + + "'source.history.enable'='false') " + + " with sink (" + + "'sink.batch.enable'='false', " + + "'sink.port'='%s'," + + "'sink.ip'='%s'," + + "'sink'='iotdb-thrift-sink')", receiverIp, receiverPort)); fail(); } catch (SQLException ignore) { @@ -156,14 +156,14 @@ public void testRevertStageOrder() throws Exception { statement.execute( String.format( "create pipe p1" - + " with connector (" - + "'connector.batch.enable'='false', " - + "'connector.port'='%s'," - + "'connector.ip'='%s'," - + "'connector'='iotdb-thrift-connector') " - + " with extractor (" - + "'extractor.realtime.mode'='hybrid'," - + "'extractor.history.enable'='false')", + + " with sink (" + + "'sink.batch.enable'='false', " + + "'sink.port'='%s'," + + "'sink.ip'='%s'," + + "'sink'='iotdb-thrift-sink') " + + " with source (" + + "'source.realtime.mode'='hybrid'," + + "'source.history.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -194,7 +194,7 @@ public void testMissingStage() throws Exception { try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("create pipe p2 with extractor ('extractor'='iotdb-extractor')"); + statement.execute("create pipe p2 with source ('source'='iotdb-source')"); fail(); } catch (SQLException ignored) { } @@ -203,7 +203,7 @@ public void testMissingStage() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( "create pipe p3" - + " with extractor ('extractor'='iotdb-extractor')" + + " with source ('source'='iotdb-source')" + " with processor ('processor'='do-nothing-processor')"); fail(); } catch (SQLException ignored) { @@ -214,11 +214,11 @@ public void testMissingStage() throws Exception { statement.execute( String.format( "create pipe p4" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -230,13 +230,13 @@ public void testMissingStage() throws Exception { statement.execute( String.format( "create pipe p5" - + " with extractor ('extractor'='iotdb-extractor')" + + " with source ('source'='iotdb-source')" + " with processor ('processor'='do-nothing-processor')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -264,13 +264,13 @@ public void testInvalidParameter() throws Exception { statement.execute( String.format( "create pipe p1" - + " with extractor ()" + + " with source ()" + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -282,13 +282,13 @@ public void testInvalidParameter() throws Exception { statement.execute( String.format( "create pipe p2" - + " with extractor ('extractor'='invalid-param')" + + " with source ('source'='invalid-param')" + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -299,13 +299,13 @@ public void testInvalidParameter() throws Exception { statement.execute( String.format( "create pipe p3" - + " with extractor ()" + + " with source ()" + " with processor ('processor'='invalid-param')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -316,13 +316,13 @@ public void testInvalidParameter() throws Exception { statement.execute( String.format( "create pipe p4" - + " with extractor ()" + + " with source ()" + " with processor ()" - + " with connector (" - + "'connector'='invalid-param'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='invalid-param'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -348,13 +348,13 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe extractor1" - + " with extractor ('extractor'='iotdb-extractor')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + "create pipe source1" + + " with source ('source'='iotdb-source')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -365,13 +365,13 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe extractor2" - + " with extractor (\"extractor\"=\"iotdb-extractor\")" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + "create pipe source2" + + " with source (\"source\"=\"iotdb-source\")" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -382,13 +382,13 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe extractor3" - + " with extractor ('extractor'=\"iotdb-extractor\")" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + "create pipe source3" + + " with source ('source'=\"iotdb-source\")" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -399,13 +399,13 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe extractor4" - + " with extractor (extractor=iotdb-extractor)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + "create pipe source4" + + " with source (source=iotdb-source)" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -415,13 +415,13 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe extractor5" - + " with extractor ('extractor'=`iotdb-extractor`)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + "create pipe source5" + + " with source ('source'=`iotdb-source`)" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -433,11 +433,11 @@ public void testBrackets() throws Exception { String.format( "create pipe processor1" + " with processor ('processor'='do-nothing-processor')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -450,11 +450,11 @@ public void testBrackets() throws Exception { String.format( "create pipe processor2" + " with processor (\"processor\"=\"do-nothing-processor\")" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -467,11 +467,11 @@ public void testBrackets() throws Exception { String.format( "create pipe processor3" + " with processor ('processor'=\"do-nothing-processor\")" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -484,11 +484,11 @@ public void testBrackets() throws Exception { String.format( "create pipe processor4" + " with processor (processor=do-nothing-processor)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -500,11 +500,11 @@ public void testBrackets() throws Exception { String.format( "create pipe processor5" + " with processor ('processor'=`do-nothing-processor`)" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -514,12 +514,12 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe connector1" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + "create pipe sink1" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -530,12 +530,12 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe connector2" - + " with connector (" - + "\"connector\"=\"iotdb-thrift-connector\"," - + "\"connector.ip\"=\"%s\"," - + "\"connector.port\"=\"%s\"," - + "\"connector.batch.enable\"=\"false\")", + "create pipe sink2" + + " with sink (" + + "\"sink\"=\"iotdb-thrift-sink\"," + + "\"sink.ip\"=\"%s\"," + + "\"sink.port\"=\"%s\"," + + "\"sink.batch.enable\"=\"false\")", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -546,12 +546,12 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe connector3" - + " with connector (" - + "'connector'=\"iotdb-thrift-connector\"," - + "\"connector.ip\"='%s'," - + "'connector.port'=\"%s\"," - + "\"connector.batch.enable\"='false')", + "create pipe sink3" + + " with sink (" + + "'sink'=\"iotdb-thrift-sink\"," + + "\"sink.ip\"='%s'," + + "'sink.port'=\"%s\"," + + "\"sink.batch.enable\"='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -562,12 +562,12 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe connector4" - + " with connector (" - + "connector=iotdb-thrift-connector," - + "connector.ip=%s," - + "connector.port=%s," - + "connector.batch.enable=false)", + "create pipe sink4" + + " with sink (" + + "sink=iotdb-thrift-sink," + + "sink.ip=%s," + + "sink.port=%s," + + "sink.batch.enable=false)", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -577,12 +577,12 @@ public void testBrackets() throws Exception { final Statement statement = connection.createStatement()) { statement.execute( String.format( - "create pipe connector5" - + " with connector (" - + "'connector'=`iotdb-thrift-connector`," - + "'connector.ip'=`%s`," - + "'connector.port'=`%s`," - + "'connector.batch.enable'=`false`)", + "create pipe sink5" + + " with sink (" + + "'sink'=`iotdb-thrift-sink`," + + "'sink.ip'=`%s`," + + "'sink.port'=`%s`," + + "'sink.batch.enable'=`false`)", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -604,35 +604,37 @@ public void testShowPipeWithWrongPipeName() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - connectorAttributes.replace("connector.batch.enable", "true"); + sinkAttributes.replace("sink.batch.enable", "true"); status = client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p3", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -647,8 +649,8 @@ public void testShowPipeWithWrongPipeName() throws Exception { Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); - // Show all pipes whose connector is also used by p1. - // p1 and p2 share the same connector parameters, so they have the same connector. + // Show all pipes whose sink is also used by p1. + // p1 and p2 share the same sink parameters, so they have the same sink. showPipeResult = client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); @@ -673,13 +675,13 @@ public void testInclusionPattern() throws Exception { statement.execute( String.format( "create pipe p2" - + " with extractor ('extractor.inclusion'='schema, auth.role', 'extractor.inclusion.exclusion'='all')" + + " with source ('source.inclusion'='schema, auth.role', 'source.inclusion.exclusion'='all')" + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -691,13 +693,13 @@ public void testInclusionPattern() throws Exception { statement.execute( String.format( "create pipe p3" - + " with extractor ('extractor.inclusion'='wrong')" + + " with source ('source.inclusion'='wrong')" + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -709,13 +711,13 @@ public void testInclusionPattern() throws Exception { statement.execute( String.format( "create pipe p4" - + " with extractor ('extractor.inclusion.exclusion'='wrong')" + + " with source ('source.inclusion.exclusion'='wrong')" + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); fail(); } catch (SQLException ignored) { @@ -727,13 +729,13 @@ public void testInclusionPattern() throws Exception { statement.execute( String.format( "create pipe p4" - + " with extractor ('extractor.inclusion'='all', 'extractor.inclusion.exclusion'='schema.database.drop, auth.role')" + + " with source ('source.inclusion'='all', 'source.inclusion.exclusion'='schema.database.drop, auth.role')" + " with processor ()" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java index 4e2871067931..23b09b23bea3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java @@ -70,22 +70,23 @@ public void testPrefixPattern() throws Exception { null); awaitUntilFlush(senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.pattern", "root.db.d1.s"); - extractorAttributes.put("extractor.inclusion", "data.insert"); + sourceAttributes.put("source.pattern", "root.db.d1.s"); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -122,22 +123,23 @@ public void testIoTDBPattern() throws Exception { null); awaitUntilFlush(senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.path", "root.**.d1.s*"); - extractorAttributes.put("extractor.inclusion", "data.insert"); + sourceAttributes.put("source.path", "root.**.d1.s*"); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -174,23 +176,24 @@ public void testIoTDBPatternWithLegacySyntax() throws Exception { null); awaitUntilFlush(senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.pattern", "root.**.d1.s*"); - extractorAttributes.put("extractor.pattern.format", "iotdb"); - extractorAttributes.put("extractor.inclusion", "data.insert"); + sourceAttributes.put("source.pattern", "root.**.d1.s*"); + sourceAttributes.put("source.pattern.format", "iotdb"); + sourceAttributes.put("source.inclusion", "data.insert"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java index d8a191c37196..f1a623143aad 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java @@ -105,24 +105,25 @@ public void testDoubleLivingAutoConflict() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); // Add this property to avoid making self cycle. - extractorAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("sink", "iotdb-thrift-sink"); - connectorAttributes.put("sink.batch.enable", "false"); - connectorAttributes.put("sink.ip", receiverIp); - connectorAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -152,24 +153,25 @@ public void testDoubleLivingAutoConflict() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); // Add this property to avoid to make self cycle. - extractorAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", senderIp); + sinkAttributes.put("sink.port", Integer.toString(senderPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -246,23 +248,24 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.inclusion.exclusion", ""); - extractorAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -272,23 +275,24 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.inclusion", "all"); - extractorAttributes.put("source.inclusion.exclusion", ""); - extractorAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderIp); - connectorAttributes.put("connector.port", Integer.toString(senderPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", senderIp); + sinkAttributes.put("sink.port", Integer.toString(senderPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -347,20 +351,21 @@ public void testAutoManualCreateRace() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "all"); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -409,20 +414,21 @@ public void testHistoricalActivationRace() throws Exception { "insert into root.sg_aligned.device_aligned.d10(time, s0, s1, s2,s3,s4,s5,s6) values(1706678800,1,1706678800,5.39,5.51234,false,'add:s6',32);"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "all"); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java index 519df6bf0a44..405c5f72371a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java @@ -74,21 +74,22 @@ public void testAutoDropInHistoricalTransfer() throws Exception { null); awaitUntilFlush(senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.mode", "query"); + sourceAttributes.put("source.mode", "query"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -146,23 +147,24 @@ public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { null); awaitUntilFlush(senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.mode", "query"); - extractorAttributes.put("extractor.start-time", "1970-01-01T08:00:02+08:00"); - extractorAttributes.put("extractor.end-time", "1970-01-01T08:00:04+08:00"); + sourceAttributes.put("source.mode", "query"); + sourceAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); + sourceAttributes.put("source.end-time", "1970-01-01T08:00:04+08:00"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java index bb0423b98099..ff06e6531cfc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java @@ -106,12 +106,12 @@ public void setUp() { @Test public void testMachineDowntimeAsync() { - testMachineDowntime("iotdb-thrift-connector"); + testMachineDowntime("iotdb-thrift-sink"); } @Test public void testMachineDowntimeSync() { - testMachineDowntime("iotdb-thrift-sync-connector"); + testMachineDowntime("iotdb-thrift-sync-sink"); } private void testMachineDowntime(String sink) { @@ -135,23 +135,24 @@ private void testMachineDowntime(String sink) { "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor", "iotdb-extractor"); - extractorAttributes.put("capture.tree", "true"); + sourceAttributes.put("source", "iotdb-source"); + sourceAttributes.put("capture.tree", "true"); + sourceAttributes.put("user", "root"); processorAttributes.put("processor", "do-nothing-processor"); - connectorAttributes.put("connector", sink); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.node-urls", a.toString()); + sinkAttributes.put("sink", sink); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.node-urls", a.toString()); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -227,31 +228,32 @@ public void testWithAllParameters(final String realtimeMode) throws Exception { "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor", "iotdb-extractor"); - extractorAttributes.put("extractor.pattern", "root.db.d1"); - extractorAttributes.put("extractor.history.enable", "true"); - extractorAttributes.put("extractor.history.start-time", "2010-01-01T08:00:00+08:00"); - extractorAttributes.put("extractor.history.end-time", "2010-01-02T08:00:00+08:00"); - extractorAttributes.put("extractor.realtime.enable", "true"); - extractorAttributes.put("extractor.realtime.mode", realtimeMode); + sourceAttributes.put("source", "iotdb-source"); + sourceAttributes.put("source.pattern", "root.db.d1"); + sourceAttributes.put("source.history.enable", "true"); + sourceAttributes.put("source.history.start-time", "2010-01-01T08:00:00+08:00"); + sourceAttributes.put("source.history.end-time", "2010-01-02T08:00:00+08:00"); + sourceAttributes.put("source.realtime.enable", "true"); + sourceAttributes.put("source.realtime.mode", realtimeMode); + sourceAttributes.put("user", "root"); processorAttributes.put("processor", "do-nothing-processor"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.user", "root"); - connectorAttributes.put("connector.password", "root"); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.user", "root"); + sinkAttributes.put("sink.password", "root"); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -286,21 +288,22 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.pattern", "root.db.d1"); + sourceAttributes.put("source.pattern", "root.db.d1"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -373,21 +376,22 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { // Create a new pipe and write new data - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.pattern", "root.db.d2"); + sourceAttributes.put("source.pattern", "root.db.d2"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -416,21 +420,22 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.pattern", "root.db.d1"); + sourceAttributes.put("source.pattern", "root.db.d1"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -472,21 +477,22 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { // create a new pipe and write new data - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.pattern", "root.db.d2"); + sourceAttributes.put("source.pattern", "root.db.d2"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -516,14 +522,16 @@ public void testCreatePipeWhenRegisteringNewDataNode() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final Thread t = new Thread( @@ -531,8 +539,8 @@ public void testCreatePipeWhenRegisteringNewDataNode() throws Exception { for (int i = 0; i < 30; ++i) { try { client.createPipe( - new TCreatePipeReq("p" + i, connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p" + i, sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); } catch (final TException e) { // Not sure if the "createPipe" has succeeded @@ -573,19 +581,21 @@ public void testRegisteringNewDataNodeWhenTransferringData() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -660,19 +670,21 @@ public void testRegisteringNewDataNodeAfterTransferringData() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -726,19 +738,21 @@ public void testNewDataNodeFailureParallelToTransferringData() throws Exception try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -790,19 +804,21 @@ public void testSenderRestartWhenTransferring() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -842,14 +858,16 @@ public void testConcurrentlyCreatePipeOfSameName() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final AtomicInteger successCount = new AtomicInteger(0); final List threads = new ArrayList<>(); @@ -861,8 +879,8 @@ public void testConcurrentlyCreatePipeOfSameName() throws Exception { (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { successCount.incrementAndGet(); @@ -946,14 +964,16 @@ private void testCreatePipesWithSameConnector(final int pipeCount) throws Except final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final AtomicInteger successCount = new AtomicInteger(0); final List threads = new ArrayList<>(); @@ -966,8 +986,8 @@ private void testCreatePipesWithSameConnector(final int pipeCount) throws Except (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { final TSStatus status = client.createPipe( - new TCreatePipeReq("p" + finalI, connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p" + finalI, sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -1020,22 +1040,23 @@ public void testNegativeTimestamp() throws Exception { "insert into root.db.d1(time, s1) values (1960-01-02T10:00:00+08:00, 2)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor", "iotdb-extractor"); + sourceAttributes.put("source", "iotdb-source"); + sourceAttributes.put("user", "root"); processorAttributes.put("processor", "do-nothing-processor"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java index 39bdd8216470..53da5d3f498a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java @@ -417,24 +417,26 @@ private void testIdempotent( try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java index 9dc07aaa6c3a..e3f6b3a95779 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java @@ -215,24 +215,26 @@ private void testInsertNullValueTemplate( try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); if (withParsing) { - extractorAttributes.put("start-time", "1970-01-01T08:00:00.000+08:00"); - extractorAttributes.put("end-time", "1970-01-01T09:00:00.000+08:00"); - extractorAttributes.put("extractor.pattern", "root.sg.d1"); + sourceAttributes.put("start-time", "1970-01-01T08:00:00.000+08:00"); + sourceAttributes.put("end-time", "1970-01-01T09:00:00.000+08:00"); + sourceAttributes.put("source.pattern", "root.sg.d1"); } final TSStatus status = client.createPipe( - new TCreatePipeReq("test", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("test", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java index 601b916ae19d..3900506a658b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java @@ -96,42 +96,42 @@ public void setUp() { @Test public void testCompression1() throws Exception { - doTest("iotdb-thrift-connector", "stream", true, "snappy"); + doTest("iotdb-thrift-sink", "stream", true, "snappy"); } @Test public void testCompression2() throws Exception { - doTest("iotdb-thrift-connector", "batch", true, "snappy, lzma2"); + doTest("iotdb-thrift-sink", "batch", true, "snappy, lzma2"); } @Test public void testCompression3() throws Exception { - doTest("iotdb-thrift-sync-connector", "stream", false, "snappy, snappy"); + doTest("iotdb-thrift-sync-sink", "stream", false, "snappy, snappy"); } @Test public void testCompression4() throws Exception { - doTest("iotdb-thrift-sync-connector", "batch", true, "gzip, zstd"); + doTest("iotdb-thrift-sync-sink", "batch", true, "gzip, zstd"); } @Test public void testCompression5() throws Exception { - doTest("iotdb-air-gap-connector", "stream", false, "lzma2, lz4"); + doTest("iotdb-air-gap-sink", "stream", false, "lzma2, lz4"); } @Test public void testCompression6() throws Exception { - doTest("iotdb-air-gap-connector", "batch", true, "lzma2"); + doTest("iotdb-air-gap-sink", "batch", true, "lzma2"); } private void doTest( - String connectorType, String realtimeMode, boolean useBatchMode, String compressionTypes) + String sinkType, String realtimeMode, boolean useBatchMode, String compressionTypes) throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); final int receiverPort = - connectorType.contains("air-gap") + sinkType.contains("air-gap") ? receiverDataNode.getPipeAirGapReceiverPort() : receiverDataNode.getPort(); @@ -150,27 +150,28 @@ private void doTest( "insert into root.db.d1(time, s1) values (2010-01-02T10:00:00+08:00, 2)", "flush"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor", "iotdb-extractor"); - extractorAttributes.put("extractor.realtime.mode", realtimeMode); + sourceAttributes.put("source", "iotdb-source"); + sourceAttributes.put("source.realtime.mode", realtimeMode); + sourceAttributes.put("user", "root"); processorAttributes.put("processor", "do-nothing-processor"); - connectorAttributes.put("connector", connectorType); - connectorAttributes.put("connector.batch.enable", useBatchMode ? "true" : "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.user", "root"); - connectorAttributes.put("connector.password", "root"); - connectorAttributes.put("connector.compressor", compressionTypes); + sinkAttributes.put("sink", sinkType); + sinkAttributes.put("sink.batch.enable", useBatchMode ? "true" : "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.user", "root"); + sinkAttributes.put("sink.password", "root"); + sinkAttributes.put("sink.compressor", compressionTypes); final TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -238,12 +239,12 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p1" - + " with extractor ('extractor.pattern'='root.db.d1.s1')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='3')", + + " with source ('source.pattern'='root.db.d1.s1')" + + " with sink (" + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.compressor'='zstd, zstd'," + + "'sink.compressor.zstd.level'='3')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -255,12 +256,12 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p2" - + " with extractor ('extractor.pattern'='root.db.d1.s2')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='22')", + + " with source ('source.pattern'='root.db.d1.s2')" + + " with sink (" + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.compressor'='zstd, zstd'," + + "'sink.compressor.zstd.level'='22')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -272,12 +273,12 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p3" - + " with extractor ('extractor.pattern'='root.db.d1.s3')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='-131072')", + + " with source ('source.pattern'='root.db.d1.s3')" + + " with sink (" + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.compressor'='zstd, zstd'," + + "'sink.compressor.zstd.level'='-131072')", receiverIp, receiverPort)); } catch (SQLException e) { e.printStackTrace(); @@ -289,12 +290,12 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p4" - + " with extractor ('extractor.pattern'='root.db.d1.s4')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='-131073')", + + " with source ('source.pattern'='root.db.d1.s4')" + + " with sink (" + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.compressor'='zstd, zstd'," + + "'sink.compressor.zstd.level'='-131073')", receiverIp, receiverPort)); fail(); } catch (SQLException e) { @@ -307,12 +308,12 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p5" - + " with extractor ('extractor.pattern'='root.db.d1.s5')" - + " with connector (" - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.compressor'='zstd, zstd'," - + "'connector.compressor.zstd.level'='23')", + + " with source ('source.pattern'='root.db.d1.s5')" + + " with sink (" + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.compressor'='zstd, zstd'," + + "'sink.compressor.zstd.level'='23')", receiverIp, receiverPort)); fail(); } catch (SQLException e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java index 74c90088fea1..26790479f22f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java @@ -95,16 +95,17 @@ public void testReceiverNotLoadDeletedTimeseries() throws Exception { final String receiverIp = receiverDataNode.getIp(); final int receiverPort = receiverDataNode.getPort(); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); // Enable mods transfer - extractorAttributes.put("source.mods.enable", "true"); + sourceAttributes.put("source.mods.enable", "true"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -138,8 +139,8 @@ public void testReceiverNotLoadDeletedTimeseries() throws Exception { TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); Assert.assertEquals( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java index cd3009dd9f8a..f2735938e16f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java @@ -67,53 +67,56 @@ public void testPipeNowFunction() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - Map extractorAttributes = new HashMap<>(); + Map sourceAttributes = new HashMap<>(); Map processorAttributes = new HashMap<>(); - Map connectorAttributes = new HashMap<>(); + Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("source.start-time", "now"); - extractorAttributes.put("source.end-time", "now"); - extractorAttributes.put("source.history.start-time", "now"); - extractorAttributes.put("source.history.end-time", "now"); - extractorAttributes.put("source.history.enable", "true"); + sourceAttributes.put("source.start-time", "now"); + sourceAttributes.put("source.end-time", "now"); + sourceAttributes.put("source.history.start-time", "now"); + sourceAttributes.put("source.history.end-time", "now"); + sourceAttributes.put("source.history.enable", "true"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); TSStatus status = client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - extractorAttributes.clear(); - extractorAttributes.put("start-time", "now"); - extractorAttributes.put("end-time", "now"); - extractorAttributes.put("history.start-time", "now"); - extractorAttributes.put("history.end-time", "now"); - extractorAttributes.put("history.enable", "true"); + sourceAttributes.clear(); + sourceAttributes.put("start-time", "now"); + sourceAttributes.put("end-time", "now"); + sourceAttributes.put("history.start-time", "now"); + sourceAttributes.put("history.end-time", "now"); + sourceAttributes.put("history.enable", "true"); + sourceAttributes.put("user", "root"); status = client.createPipe( - new TCreatePipeReq("p2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - extractorAttributes.clear(); - extractorAttributes.put("extractor.start-time", "now"); - extractorAttributes.put("extractor.end-time", "now"); - extractorAttributes.put("extractor.history.start-time", "now"); - extractorAttributes.put("extractor.history.end-time", "now"); - extractorAttributes.put("history.enable", "true"); + sourceAttributes.clear(); + sourceAttributes.put("source.start-time", "now"); + sourceAttributes.put("source.end-time", "now"); + sourceAttributes.put("source.history.start-time", "now"); + sourceAttributes.put("source.history.end-time", "now"); + sourceAttributes.put("history.enable", "true"); + sourceAttributes.put("user", "root"); status = client.createPipe( - new TCreatePipeReq("p3", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("p3", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -132,15 +135,16 @@ public void testPipeNowFunction() throws Exception { Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - extractorAttributes.clear(); - extractorAttributes.put("extractor.start-time", "now"); - extractorAttributes.put("extractor.end-time", "now"); - extractorAttributes.put("extractor.history.start-time", "now"); - extractorAttributes.put("extractor.history.end-time", "now"); + sourceAttributes.clear(); + sourceAttributes.put("source.start-time", "now"); + sourceAttributes.put("source.end-time", "now"); + sourceAttributes.put("source.history.start-time", "now"); + sourceAttributes.put("source.history.end-time", "now"); + sourceAttributes.put("user", "root"); client.alterPipe( new TAlterPipeReq() .setPipeName("p1") - .setExtractorAttributes(extractorAttributes) + .setExtractorAttributes(sourceAttributes) .setIsReplaceAllExtractorAttributes(false) .setProcessorAttributes(new HashMap<>()) .setIsReplaceAllProcessorAttributes(false) @@ -153,15 +157,16 @@ public void testPipeNowFunction() throws Exception { Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - extractorAttributes.clear(); - extractorAttributes.put("start-time", "now"); - extractorAttributes.put("end-time", "now"); - extractorAttributes.put("history.start-time", "now"); - extractorAttributes.put("history.end-time", "now"); + sourceAttributes.clear(); + sourceAttributes.put("start-time", "now"); + sourceAttributes.put("end-time", "now"); + sourceAttributes.put("history.start-time", "now"); + sourceAttributes.put("history.end-time", "now"); + sourceAttributes.put("user", "root"); client.alterPipe( new TAlterPipeReq() .setPipeName("p1") - .setExtractorAttributes(extractorAttributes) + .setExtractorAttributes(sourceAttributes) .setIsReplaceAllExtractorAttributes(false) .setProcessorAttributes(new HashMap<>()) .setIsReplaceAllProcessorAttributes(false) @@ -192,17 +197,17 @@ private void doTest(String dialect) { final String p1 = String.format( "create pipe p1" - + " with extractor (" - + "'extractor.history.enable'='true'," + + " with source (" + + "'source.history.enable'='true'," + "'source.start-time'='now'," + "'source.end-time'='now'," + "'source.history.start-time'='now'," + "'source.history.end-time'='now')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(dialect); final Statement statement = connection.createStatement()) { @@ -214,17 +219,17 @@ private void doTest(String dialect) { final String p2 = String.format( "create pipe p2" - + " with extractor (" - + "'extractor.history.enable'='true'," + + " with source (" + + "'source.history.enable'='true'," + "'start-time'='now'," + "'end-time'='now'," + "'history.start-time'='now'," + "'history.end-time'='now')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(dialect); final Statement statement = connection.createStatement()) { @@ -236,17 +241,17 @@ private void doTest(String dialect) { final String p3 = String.format( "create pipe p3" - + " with extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.start-time'='now'," - + "'extractor.end-time'='now'," - + "'extractor.history.start-time'='now'," - + "'extractor.history.end-time'='now')" - + " with connector (" - + "'connector'='iotdb-thrift-connector'," - + "'connector.ip'='%s'," - + "'connector.port'='%s'," - + "'connector.batch.enable'='false')", + + " with source (" + + "'source.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')" + + " with sink (" + + "'sink'='iotdb-thrift-sink'," + + "'sink.ip'='%s'," + + "'sink.port'='%s'," + + "'sink.batch.enable'='false')", receiverIp, receiverPort); try (final Connection connection = senderEnv.getConnection(dialect); final Statement statement = connection.createStatement()) { @@ -257,7 +262,7 @@ private void doTest(String dialect) { String alterP3 = "alter pipe p3" - + " modify extractor (" + + " modify source (" + "'history.enable'='true'," + "'start-time'='now'," + "'end-time'='now'," @@ -272,12 +277,12 @@ private void doTest(String dialect) { alterP3 = "alter pipe p3" - + " modify extractor (" - + "'extractor.history.enable'='true'," - + "'extractor.start-time'='now'," - + "'extractor.end-time'='now'," - + "'extractor.history.start-time'='now'," - + "'extractor.history.end-time'='now')"; + + " modify source (" + + "'source.history.enable'='true'," + + "'source.start-time'='now'," + + "'source.end-time'='now'," + + "'source.history.start-time'='now'," + + "'source.history.end-time'='now')"; try (final Connection connection = senderEnv.getConnection(dialect); final Statement statement = connection.createStatement()) { statement.execute(alterP3); @@ -288,7 +293,7 @@ private void doTest(String dialect) { alterP3 = "alter pipe p3" + " modify source (" - + "'extractor.history.enable'='true'," + + "'source.history.enable'='true'," + "'source.start-time'='now'," + "'source.end-time'='now'," + "'source.history.start-time'='now'," diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java index 84c90a667eec..f04c2ae65ad3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java @@ -111,21 +111,22 @@ public void testPureSchemaInclusionWithMultiplePattern() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "schema"); - extractorAttributes.put("path", "root.ln.wf01.wt01.status,root.ln.wf02.**"); + sourceAttributes.put("path", "root.ln.wf01.wt01.status,root.ln.wf02.**"); + sourceAttributes.put("source.inclusion", "schema"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -182,12 +183,13 @@ public void testAuthExclusion() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", "auth"); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", "auth"); + sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -196,7 +198,7 @@ public void testAuthExclusion() throws Exception { final TSStatus status = client.createPipe( new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(extractorAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -220,12 +222,13 @@ public void testAuthInclusionWithPattern() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "auth"); - extractorAttributes.put("path", "root.ln.**"); + sourceAttributes.put("source.inclusion", "auth"); + sourceAttributes.put("path", "root.ln.**"); + sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -234,7 +237,7 @@ public void testAuthInclusionWithPattern() throws Exception { final TSStatus status = client.createPipe( new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(extractorAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -272,11 +275,12 @@ public void testPureDeleteInclusion() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "data.delete"); + sourceAttributes.put("source.inclusion", "data.delete"); + sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -285,7 +289,7 @@ public void testPureDeleteInclusion() throws Exception { final TSStatus status = client.createPipe( new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(extractorAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java index cc391f39b437..76b008166067 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java @@ -51,24 +51,25 @@ public void testDoubleLivingTimeseries() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "data, schema"); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.inclusion", "data, schema"); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -79,25 +80,25 @@ public void testDoubleLivingTimeseries() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "data, schema"); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.inclusion", "data, schema"); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderEnv.getDataNodeWrapper(0).getIp()); - connectorAttributes.put( - "connector.port", Integer.toString(senderEnv.getDataNodeWrapper(0).getPort())); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", senderEnv.getDataNodeWrapper(0).getIp()); + sinkAttributes.put("sink.port", Integer.toString(senderEnv.getDataNodeWrapper(0).getPort())); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -159,24 +160,25 @@ public void testDoubleLivingTemplate() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "data, schema"); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.inclusion", "data, schema"); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -187,25 +189,25 @@ public void testDoubleLivingTemplate() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) receiverEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "data, schema"); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.inclusion", "data, schema"); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", senderEnv.getDataNodeWrapper(0).getIp()); - connectorAttributes.put( - "connector.port", Integer.toString(senderEnv.getDataNodeWrapper(0).getPort())); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); + sinkAttributes.put("sink.batch.enable", "false"); + sinkAttributes.put("sink.ip", senderEnv.getDataNodeWrapper(0).getIp()); + sinkAttributes.put("sink.port", Integer.toString(senderEnv.getDataNodeWrapper(0).getPort())); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java index cb0c334f68b9..5f936cda5161 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java @@ -116,24 +116,25 @@ public void testTemplateInclusion() throws Exception { null); awaitUntilFlush(senderEnv); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "data, schema"); - extractorAttributes.put("extractor.inclusion.exclusion", "schema.timeseries.ordinary"); - extractorAttributes.put("extractor.path", "root.ln.**"); + sourceAttributes.put("source.inclusion", "data, schema"); + sourceAttributes.put("source.inclusion.exclusion", "schema.timeseries.ordinary"); + sourceAttributes.put("source.path", "root.ln.**"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -203,22 +204,23 @@ public void testAuthInclusion() throws Exception { "insert into root.ln.wf01.wt01(time, temperature, status) values (1800000000000, 23, true)"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "auth"); + sourceAttributes.put("source.inclusion", "auth"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -287,22 +289,23 @@ public void testTimeSeriesInclusion() throws Exception { "create aligned timeseries root.sg.`apache|timecho-tag-attr`.d1(s1 INT32 tags(tag1=v1, tag2=v2) attributes(attr1=v1, attr2=v2), s2 DOUBLE tags(tag3=v3, tag4=v4) attributes(attr3=v3, attr4=v4))"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "schema"); + sourceAttributes.put("source.inclusion", "schema"); + sourceAttributes.put("user", "root"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java index f890a2e8cc51..119f7ed59c2f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java @@ -75,23 +75,25 @@ public void testConfigNodeLeaderChange() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -128,23 +130,25 @@ public void testSchemaRegionLeaderChange() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java index 0d21879801ac..053fdee0731a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java @@ -50,23 +50,25 @@ public void testAutoRestartSchemaTask() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); @@ -117,23 +119,25 @@ public void testAutoRestartConfigTask() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); + + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java index a0d5ee7e1d13..faafc5bf9139 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java @@ -58,23 +58,25 @@ public void testMultiSchemaRegion() throws Exception { "create timeseries root.sg.wf01.GPS.status0 with datatype=BOOLEAN,encoding=PLAIN"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("source.inclusion.exclusion", ""); + sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("user", "root"); + + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverIp); + sinkAttributes.put("sink.port", Integer.toString(receiverPort)); + sinkAttributes.put("sink.exception.conflict.resolve-strategy", "retry"); + sinkAttributes.put("sink.exception.conflict.retry-max-time-seconds", "-1"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java index 7fa5bdc0b376..2d5ea28ea1c2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java @@ -22,12 +22,14 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -36,6 +38,9 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -43,6 +48,8 @@ import java.util.Map; import java.util.Set; +import static org.junit.Assert.fail; + @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) public class IoTDBPipePermissionIT extends AbstractPipeDualTreeModelManualIT { @@ -126,6 +133,7 @@ private void testWithSink(final String sink) throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("user", "root"); sinkAttributes.put("sink", sink); sinkAttributes.put("sink.ip", receiverIp); @@ -198,6 +206,7 @@ public void testNoPermission() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("source.inclusion", "all"); + sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-async-sink"); sinkAttributes.put("sink.ip", receiverIp); @@ -224,4 +233,156 @@ public void testNoPermission() throws Exception { receiverEnv, "list user", "User,", Collections.singleton("root,")); } } + + @Test + public void testSourcePermission() { + TestUtils.executeNonQuery(senderEnv, "create user `thulab` 'passwD@123456'", null); + + // Shall fail if username is specified without password + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'user'='thulab')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + fail("When the 'user' or 'username' is specified, password must be specified too."); + } catch (final SQLException ignore) { + // Expected + } + + // Shall fail if password is wrong + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'user'='thulab'" + + "'password'='hack')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + fail("Shall fail if password is wrong."); + } catch (final SQLException ignore) { + // Expected + } + + // Use current session, user is root + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2b" + + " with source (" + + "'inclusion'='all')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Create pipe without user shall succeed if use the current session"); + } + + // Alter to another user, shall fail because of lack of password + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ('username'='thulab')"); + fail("Alter pipe shall fail if only user is specified"); + } catch (final SQLException ignore) { + // Expected + } + + // Successfully alter + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + "alter pipe a2b modify source ('username'='thulab', 'password'='passwD@123456')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Alter pipe shall not fail if user and password are specified"); + } + + TestUtils.executeNonQuery(senderEnv, "create database root.test"); + + // Shall not be transferred + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "count databases", "count,", Collections.singleton("0,")); + + // GRANT privileges ON prefixPath (COMMA prefixPath)* TO USER userName=usernameWithRoot + // (grantOpt)? + // Grant some privilege + TestUtils.executeNonQuery(senderEnv, "grant SYSTEM on root.** to user thulab"); + + TestUtils.executeNonQuery(senderEnv, "create database root.test1"); + + // Shall be transferred + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count databases root.tes*", "count,", Collections.singleton("1,")); + + // Alter pipe, throw exception if no privileges + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ('skipif'='')"); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Write some data + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "create timeSeries root.vehicle.car.temperature DOUBLE", + "insert into root.vehicle.car(temperature) values (36.5)")); + + // Exception, block here + TableModelUtils.assertCountDataAlwaysOnEnv("test", "test", 0, receiverEnv); + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, "count timeSeries", "count(timeseries),", Collections.singleton("0,")); + + // Grant SELECT privilege + TestUtils.executeNonQueries( + senderEnv, Arrays.asList("grant READ on root.** to user thulab", "start pipe a2b")); + + // Will finally pass + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.vehicle.**", + "count(root.vehicle.car.temperature),", + Collections.singleton("1,")); + + // test showing pipe + // Create another pipe, user is root + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe a2c" + + " with source (" + + "'inclusion'='all'," + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail("Create pipe without user shall succeed if use the current session"); + } + + TestUtils.executeNonQuery(senderEnv, "revoke SYSTEM on root.** from user thulab"); + + // A user shall only see its own pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + Assert.assertEquals( + 1, client.showPipe(new TShowPipeReq().setUserName("thulab")).pipeInfoList.size()); + } catch (Exception e) { + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java index 8be50230144c..9430e0adbfa4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java @@ -28,6 +28,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -41,6 +42,7 @@ @Category({MultiClusterIT1.class}) public class IoTDBPipeAggregateIT extends AbstractPipeSingleIT { @Test + @Ignore public void testAggregator() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { @@ -55,11 +57,11 @@ public void testAggregator() throws Exception { "insert into root.ln.wf01.wt01(time, temperature, status) values (10000, 1, false)"), null); - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); + final Map sinkAttributes = new HashMap<>(); - extractorAttributes.put("pattern", "root.ln"); + sourceAttributes.put("pattern", "root.ln"); processorAttributes.put("processor", "aggregate-processor"); processorAttributes.put("output.database", "root.testdb"); @@ -69,12 +71,12 @@ public void testAggregator() throws Exception { "operators", "avg, peak, rms, var, skew, kurt, ff, cf, pf, cE, max, min"); processorAttributes.put("sliding.seconds", "60"); - connectorAttributes.put("sink", "write-back-sink"); + sinkAttributes.put("sink", "write-back-sink"); final TSStatus status = client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("testPipe", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java index 33ab3d7a6a6d..eca7145dd344 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java @@ -49,7 +49,8 @@ public void testSinkPermission() { // Shall fail if username is specified without password try (final Connection connection = env.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - statement.execute("create pipe a2b ('user'='thulab', 'sink'='write-back-sink')"); + statement.execute( + "create pipe a2b with source ('capture.tree'='true') with sink ('user'='thulab', 'sink'='write-back-sink')"); fail("When the 'user' or 'username' is specified, password must be specified too."); } catch (final SQLException ignore) { // Expected diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java index b4818ac67f61..0beacaf85073 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java @@ -140,21 +140,21 @@ protected void setUpConfig() { public void setUp() throws Exception { super.setUp(); - // Setup connector attributes + // Setup sink attributes ASYNC_CONNECTOR_ATTRIBUTES = new HashMap<>(); - ASYNC_CONNECTOR_ATTRIBUTES.put("connector", "iotdb-thrift-async-connector"); - ASYNC_CONNECTOR_ATTRIBUTES.put("connector.ip", receiverEnv.getIP()); - ASYNC_CONNECTOR_ATTRIBUTES.put("connector.port", receiverEnv.getPort()); + ASYNC_CONNECTOR_ATTRIBUTES.put("sink", "iotdb-thrift-async-sink"); + ASYNC_CONNECTOR_ATTRIBUTES.put("sink.ip", receiverEnv.getIP()); + ASYNC_CONNECTOR_ATTRIBUTES.put("sink.port", receiverEnv.getPort()); SYNC_CONNECTOR_ATTRIBUTES = new HashMap<>(); - SYNC_CONNECTOR_ATTRIBUTES.put("connector", "iotdb-thrift-sync-connector"); - SYNC_CONNECTOR_ATTRIBUTES.put("connector.ip", receiverEnv.getIP()); - SYNC_CONNECTOR_ATTRIBUTES.put("connector.port", receiverEnv.getPort()); + SYNC_CONNECTOR_ATTRIBUTES.put("sink", "iotdb-thrift-sync-sink"); + SYNC_CONNECTOR_ATTRIBUTES.put("sink.ip", receiverEnv.getIP()); + SYNC_CONNECTOR_ATTRIBUTES.put("sink.port", receiverEnv.getPort()); LEGACY_CONNECTOR_ATTRIBUTES = new HashMap<>(); - LEGACY_CONNECTOR_ATTRIBUTES.put("connector", "iotdb-legacy-pipe-connector"); - LEGACY_CONNECTOR_ATTRIBUTES.put("connector.ip", receiverEnv.getIP()); - LEGACY_CONNECTOR_ATTRIBUTES.put("connector.port", receiverEnv.getPort()); + LEGACY_CONNECTOR_ATTRIBUTES.put("sink", "iotdb-legacy-pipe-sink"); + LEGACY_CONNECTOR_ATTRIBUTES.put("sink.ip", receiverEnv.getIP()); + LEGACY_CONNECTOR_ATTRIBUTES.put("sink.port", receiverEnv.getPort()); final StringBuilder nodeUrlsBuilder = new StringBuilder(); for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { @@ -166,8 +166,8 @@ public void setUp() throws Exception { .append(","); } AIR_GAP_CONNECTOR_ATTRIBUTES = new HashMap<>(); - AIR_GAP_CONNECTOR_ATTRIBUTES.put("connector", "iotdb-air-gap-connector"); - AIR_GAP_CONNECTOR_ATTRIBUTES.put("connector.node-urls", nodeUrlsBuilder.toString()); + AIR_GAP_CONNECTOR_ATTRIBUTES.put("sink", "iotdb-air-gap-sink"); + AIR_GAP_CONNECTOR_ATTRIBUTES.put("sink.node-urls", nodeUrlsBuilder.toString()); // Setup subscription info list with expected results { @@ -297,7 +297,7 @@ public void setUp() throws Exception { } private void testSubscriptionHistoricalDataTemplate( - final Map connectorAttributes, + final Map sinkAttributes, final List subscriptionInfoList, final Map expectedHeaderWithResult) throws Exception { @@ -310,8 +310,8 @@ private void testSubscriptionHistoricalDataTemplate( // Create topics createTopics(currentTime); - // Create pipes with given connector attributes - createPipes(currentTime, connectorAttributes); + // Create pipes with given sink attributes + createPipes(currentTime, sinkAttributes); // Create subscription and check result pollMessagesAndCheck( @@ -331,7 +331,7 @@ private void testSubscriptionHistoricalDataTemplate( } private void testSubscriptionRealtimeDataTemplate( - final Map connectorAttributes, + final Map sinkAttributes, final List subscriptionInfoList, final Map expectedHeaderWithResult) throws Exception { @@ -341,8 +341,8 @@ private void testSubscriptionRealtimeDataTemplate( // Create topics createTopics(currentTime); - // Create pipes with given connector attributes - createPipes(currentTime, connectorAttributes); + // Create pipes with given sink attributes + createPipes(currentTime, sinkAttributes); // Insert some realtime data insertData(currentTime); @@ -908,22 +908,23 @@ private void insertData(final long currentTime) { } } - private void createPipes(final long currentTime, final Map connectorAttributes) { + private void createPipes(final long currentTime, final Map sinkAttributes) { // For sync reference try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("inclusion.exclusion", "data.delete"); - extractorAttributes.put("path", "root.topic1.s"); - extractorAttributes.put("end-time", String.valueOf(currentTime - 1)); + sourceAttributes.put("inclusion", "data.insert"); + sourceAttributes.put("inclusion.exclusion", "data.delete"); + sourceAttributes.put("path", "root.topic1.s"); + sourceAttributes.put("end-time", String.valueOf(currentTime - 1)); + sourceAttributes.put("user", "root"); final TSStatus status = client.createPipe( - new TCreatePipeReq("sync_topic1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("sync_topic1", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); } catch (final Exception e) { @@ -933,18 +934,19 @@ private void createPipes(final long currentTime, final Map conne try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); + final Map sourceAttributes = new HashMap<>(); final Map processorAttributes = new HashMap<>(); - extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("inclusion.exclusion", "data.delete"); - extractorAttributes.put("path", "root.topic2.s"); - extractorAttributes.put("start-time", String.valueOf(currentTime)); + sourceAttributes.put("inclusion", "data.insert"); + sourceAttributes.put("inclusion.exclusion", "data.delete"); + sourceAttributes.put("path", "root.topic2.s"); + sourceAttributes.put("start-time", String.valueOf(currentTime)); + sourceAttributes.put("user", "root"); final TSStatus status = client.createPipe( - new TCreatePipeReq("sync_topic2", connectorAttributes) - .setExtractorAttributes(extractorAttributes) + new TCreatePipeReq("sync_topic2", sinkAttributes) + .setExtractorAttributes(sourceAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); } catch (final Exception e) { diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 58f77da6c33e..ca79f84bfcb4 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -571,21 +571,21 @@ removeAINode // Pipe Task ========================================================================================= createPipe : CREATE PIPE (IF NOT EXISTS)? pipeName=identifier - ((extractorAttributesClause? + ((sourceAttributesClause? processorAttributesClause? - connectorAttributesClause) - |connectorAttributesWithoutWithSinkClause) + sinkAttributesClause) + |sinkAttributesWithoutWithSinkClause) ; -extractorAttributesClause +sourceAttributesClause : WITH (EXTRACTOR | SOURCE) LR_BRACKET - (extractorAttributeClause COMMA)* extractorAttributeClause? + (sourceAttributeClause COMMA)* sourceAttributeClause? RR_BRACKET ; -extractorAttributeClause - : extractorKey=STRING_LITERAL OPERATOR_SEQ extractorValue=STRING_LITERAL +sourceAttributeClause + : sourceKey=STRING_LITERAL OPERATOR_SEQ sourceValue=STRING_LITERAL ; processorAttributesClause @@ -599,32 +599,32 @@ processorAttributeClause : processorKey=STRING_LITERAL OPERATOR_SEQ processorValue=STRING_LITERAL ; -connectorAttributesClause +sinkAttributesClause : WITH (CONNECTOR | SINK) LR_BRACKET - (connectorAttributeClause COMMA)* connectorAttributeClause? + (sinkAttributeClause COMMA)* sinkAttributeClause? RR_BRACKET ; -connectorAttributesWithoutWithSinkClause - : LR_BRACKET (connectorAttributeClause COMMA)* connectorAttributeClause? RR_BRACKET +sinkAttributesWithoutWithSinkClause + : LR_BRACKET (sinkAttributeClause COMMA)* sinkAttributeClause? RR_BRACKET ; -connectorAttributeClause - : connectorKey=STRING_LITERAL OPERATOR_SEQ connectorValue=STRING_LITERAL +sinkAttributeClause + : sinkKey=STRING_LITERAL OPERATOR_SEQ sinkValue=STRING_LITERAL ; alterPipe : ALTER PIPE (IF EXISTS)? pipeName=identifier - alterExtractorAttributesClause? + alterSourceAttributesClause? alterProcessorAttributesClause? - alterConnectorAttributesClause? + alterSinkAttributesClause? ; -alterExtractorAttributesClause +alterSourceAttributesClause : (MODIFY | REPLACE) (EXTRACTOR | SOURCE) LR_BRACKET - (extractorAttributeClause COMMA)* extractorAttributeClause? + (sourceAttributeClause COMMA)* sourceAttributeClause? RR_BRACKET ; @@ -635,10 +635,10 @@ alterProcessorAttributesClause RR_BRACKET ; -alterConnectorAttributesClause +alterSinkAttributesClause : (MODIFY | REPLACE) (CONNECTOR | SINK) LR_BRACKET - (connectorAttributeClause COMMA)* connectorAttributeClause? + (sinkAttributeClause COMMA)* sinkAttributeClause? RR_BRACKET ; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeDeactivateTemplatePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeDeactivateTemplatePlan.java index b91d7ab8c88d..6b09f7c9fefc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeDeactivateTemplatePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeDeactivateTemplatePlan.java @@ -21,9 +21,9 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/template/CreateSchemaTemplatePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/template/CreateSchemaTemplatePlan.java index bd8b5a7400f0..fa27754eecda 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/template/CreateSchemaTemplatePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/template/CreateSchemaTemplatePlan.java @@ -19,9 +19,9 @@ package org.apache.iotdb.confignode.consensus.request.write.template; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java index 5b5393eaa689..fd5587629eb1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java @@ -104,15 +104,9 @@ public PipeTableResp filter(final Boolean whereClause, final String pipeName) { } public PipeTableResp filter( - final Boolean whereClause, - final String pipeName, - final boolean isTableModel, - final String userName) { + final Boolean whereClause, final String pipeName, final String userName) { final PipeTableResp resp = filter(whereClause, pipeName); - resp.allPipeMeta.removeIf( - meta -> - !meta.getStaticMeta().visibleUnder(isTableModel) - || !isVisible4User(userName, meta.getStaticMeta())); + resp.allPipeMeta.removeIf(meta -> !isVisible4User(userName, meta.getStaticMeta())); return resp; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/template/TemplateInfoResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/template/TemplateInfoResp.java index fdf4c805a109..5f98894be398 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/template/TemplateInfoResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/template/TemplateInfoResp.java @@ -20,8 +20,8 @@ package org.apache.iotdb.confignode.consensus.response.template; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.consensus.common.DataSet; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.List; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/template/TemplateSetInfoResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/template/TemplateSetInfoResp.java index 08e64c2afd04..2bd10ebabf49 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/template/TemplateSetInfoResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/template/TemplateSetInfoResp.java @@ -21,8 +21,8 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.consensus.common.DataSet; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.List; import java.util.Map; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 233c3be881bd..0089cfe0db8d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -63,6 +63,7 @@ import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.ttl.TTLCache; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.AuthUtils; @@ -261,7 +262,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelInfoReq; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.exception.ConsensusException; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; import org.apache.iotdb.db.schemaengine.template.alter.TemplateAlterOperationUtil; import org.apache.iotdb.rpc.RpcUtils; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java index 087c76690587..f7ee6069e873 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java @@ -22,7 +22,9 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.auth.entity.PrivilegeUnion; +import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan; @@ -127,6 +129,11 @@ public TAuthizedPatternTreeResp fetchAuthorizedPTree(String username, int permis return authorInfo.generateAuthorizedPTree(username, permission); } + public PathPatternTree fetchRawAuthorizedPTree(final String userName, final PrivilegeType type) + throws AuthException { + return authorInfo.generateRawAuthorizedPTree(userName, type); + } + public TPermissionInfoResp checkUserPrivilegeGrantOpt(String username, PrivilegeUnion union) throws AuthException { union.setGrantOption(true); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 3dd3a2c68ffb..1efaa9a583d2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -37,6 +37,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.trigger.TriggerInformation; @@ -149,7 +150,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.exception.BatchProcessException; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskStage.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskStage.java index 670062be6ba6..b5fba6c004b4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskStage.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskStage.java @@ -32,9 +32,9 @@ public class PipeConfigNodeTaskStage extends PipeTaskStage { public PipeConfigNodeTaskStage( final String pipeName, final long creationTime, - final Map extractorAttributes, + final Map sourceAttributes, final Map processorAttributes, - final Map connectorAttributes, + final Map sinkAttributes, final PipeTaskMeta pipeTaskMeta) { try { @@ -42,9 +42,9 @@ public PipeConfigNodeTaskStage( new PipeConfigNodeSubtask( pipeName, creationTime, - extractorAttributes, + sourceAttributes, processorAttributes, - connectorAttributes, + sinkAttributes, pipeTaskMeta); } catch (final Exception e) { throw new PipeException( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index c68c6ef6fa58..4ee7680a7704 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -241,7 +241,7 @@ public TSStatus dropPipe(TDropPipeReq req) { public TShowPipeResp showPipes(final TShowPipeReq req) { try { return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2())) - .filter(req.whereClause, req.pipeName, req.isTableModel, req.userName) + .filter(req.whereClause, req.pipeName, req.userName) .convertToTShowPipeResp(); } catch (final ConsensusException e) { LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java index 252f7331d0cd..926c1f416e90 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/IoTDBConfigRegionSource.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeUnion; import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; @@ -58,26 +59,31 @@ import java.io.IOException; import java.nio.file.Paths; +import java.util.Collections; import java.util.HashSet; import java.util.Objects; import java.util.Optional; import java.util.Set; +import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + @TreeModel @TableModel public class IoTDBConfigRegionSource extends IoTDBNonDataRegionSource { - public static final PipeConfigPhysicalPlanTreePatternParseVisitor TREE_PATTERN_PARSE_VISITOR = - new PipeConfigPhysicalPlanTreePatternParseVisitor(); - public static final PipeConfigPhysicalPlanTablePatternParseVisitor TABLE_PATTERN_PARSE_VISITOR = - new PipeConfigPhysicalPlanTablePatternParseVisitor(); - public static final PipeConfigPhysicalPlanTreeScopeParseVisitor TREE_SCOPE_PARSE_VISITOR = - new PipeConfigPhysicalPlanTreeScopeParseVisitor(); - public static final PipeConfigPhysicalPlanTableScopeParseVisitor TABLE_SCOPE_PARSE_VISITOR = - new PipeConfigPhysicalPlanTableScopeParseVisitor(); - public static final PipeConfigPhysicalPlanTablePrivilegeParseVisitor - TABLE_PRIVILEGE_PARSE_VISITOR = new PipeConfigPhysicalPlanTablePrivilegeParseVisitor(); - + public static final PipeConfigTreePatternParseVisitor TREE_PATTERN_PARSE_VISITOR = + new PipeConfigTreePatternParseVisitor(); + public static final PipeConfigTablePatternParseVisitor TABLE_PATTERN_PARSE_VISITOR = + new PipeConfigTablePatternParseVisitor(); + public static final PipeConfigTreeScopeParseVisitor TREE_SCOPE_PARSE_VISITOR = + new PipeConfigTreeScopeParseVisitor(); + public static final PipeConfigTableScopeParseVisitor TABLE_SCOPE_PARSE_VISITOR = + new PipeConfigTableScopeParseVisitor(); + public static final PipeConfigTablePrivilegeParseVisitor TABLE_PRIVILEGE_PARSE_VISITOR = + new PipeConfigTablePrivilegeParseVisitor(); + // Local for exception + private PipeConfigTreePrivilegeParseVisitor treePrivilegeParseVisitor; private Set listenedTypeSet = new HashSet<>(); private CNPhysicalPlanGenerator parser; @@ -96,6 +102,7 @@ public void customize( super.customize(parameters, configuration); listenedTypeSet = ConfigRegionListeningFilter.parseListeningPlanTypeSet(parameters); + treePrivilegeParseVisitor = new PipeConfigTreePrivilegeParseVisitor(skipIfNoPrivileges); PipeConfigRegionSourceMetrics.getInstance().register(this); PipeConfigNodeRemainingTimeMetrics.getInstance().register(this); @@ -136,8 +143,8 @@ public synchronized EnrichedEvent supply() throws Exception { @Override protected long getMaxBlockingTimeMs() { - // The connector continues to submit and relies on the queue to sleep if empty - // Here we return with block to be consistent with the dataNode connector + // The sink continues to submit and relies on the queue to sleep if empty + // Here we return with block to be consistent with the dataNode sink return PipeConfig.getInstance().getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(); } @@ -183,6 +190,22 @@ userName, new PrivilegeUnion(PrivilegeType.MANAGE_USER)) case SCHEMA: // Currently do not check tree model mTree return Objects.nonNull(((PipeConfigRegionSnapshotEvent) event).getTemplateFile()) + && (permissionManager + .checkUserPrivileges( + userName, + new PrivilegeUnion( + Collections.singletonList( + new PartialPath( + new String[] {PATH_ROOT, MULTI_LEVEL_PATH_WILDCARD})), + PrivilegeType.READ_SCHEMA)) + .getStatus() + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || permissionManager + .checkUserPrivileges(userName, new PrivilegeUnion(PrivilegeType.SYSTEM)) + .getStatus() + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode()) || Objects.nonNull(userName) && permissionManager .checkUserPrivileges(userName, new PrivilegeUnion(null, false, true)) @@ -223,15 +246,20 @@ protected Optional trimRealtimeEventByPrivilege( final ConfigPhysicalPlan plan = ((PipeConfigRegionWritePlanEvent) event).getConfigPhysicalPlan(); final Boolean isTableDatabasePlan = isTableDatabasePlan(plan); - if (Boolean.FALSE.equals(isTableDatabasePlan)) { - return Optional.of(event); + if (!Boolean.TRUE.equals(isTableDatabasePlan)) { + final Optional result = treePrivilegeParseVisitor.process(plan, userName); + if (result.isPresent()) { + return Optional.of( + new PipeConfigRegionWritePlanEvent(result.get(), event.isGeneratedByPipe())); + } } - - final Optional result = - TABLE_PRIVILEGE_PARSE_VISITOR.process(plan, userName); - if (result.isPresent()) { - return Optional.of( - new PipeConfigRegionWritePlanEvent(result.get(), event.isGeneratedByPipe())); + if (!Boolean.FALSE.equals(isTableDatabasePlan)) { + final Optional result = + TABLE_PRIVILEGE_PARSE_VISITOR.process(plan, userName); + if (result.isPresent()) { + return Optional.of( + new PipeConfigRegionWritePlanEvent(result.get(), event.isGeneratedByPipe())); + } } if (skipIfNoPrivileges) { return Optional.empty(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTablePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitor.java similarity index 99% rename from iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTablePatternParseVisitor.java rename to iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitor.java index 7d121a584229..6baf78f60ac0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTablePatternParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitor.java @@ -40,7 +40,7 @@ import java.util.Optional; -public class PipeConfigPhysicalPlanTablePatternParseVisitor +public class PipeConfigTablePatternParseVisitor extends ConfigPhysicalPlanVisitor, TablePattern> { @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTablePrivilegeParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePrivilegeParseVisitor.java similarity index 99% rename from iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTablePrivilegeParseVisitor.java rename to iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePrivilegeParseVisitor.java index 40ab6d7a0968..bf0871f00323 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTablePrivilegeParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePrivilegeParseVisitor.java @@ -42,7 +42,7 @@ import java.util.Optional; -public class PipeConfigPhysicalPlanTablePrivilegeParseVisitor +public class PipeConfigTablePrivilegeParseVisitor extends ConfigPhysicalPlanVisitor, String> { @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTableScopeParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTableScopeParseVisitor.java similarity index 98% rename from iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTableScopeParseVisitor.java rename to iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTableScopeParseVisitor.java index e94511fd41e6..c3f481781c5e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTableScopeParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTableScopeParseVisitor.java @@ -32,7 +32,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class PipeConfigPhysicalPlanTableScopeParseVisitor +public class PipeConfigTableScopeParseVisitor extends ConfigPhysicalPlanVisitor, Void> { @Override public Optional visitPlan(final ConfigPhysicalPlan plan, final Void context) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitor.java similarity index 97% rename from iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java rename to iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitor.java index 339ea2755898..40eb1b4935aa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitor.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; @@ -39,7 +40,6 @@ import org.apache.iotdb.confignode.consensus.request.write.template.ExtendSchemaTemplatePlan; import org.apache.iotdb.confignode.manager.pipe.event.PipeConfigRegionWritePlanEvent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; import org.apache.tsfile.utils.Pair; @@ -58,7 +58,7 @@ import java.util.stream.Stream; /** - * The {@link PipeConfigPhysicalPlanTreePatternParseVisitor} will transform the schema {@link + * The {@link PipeConfigTreePatternParseVisitor} will transform the schema {@link * ConfigPhysicalPlan}s using {@link UnionIoTDBTreePattern}. Rule: * *

1. All patterns in the output {@link ConfigPhysicalPlan} will be the intersection of the @@ -72,10 +72,10 @@ *

4. The output {@link PlanNode} shall be a copied form of the original one because the original * one is used in the {@link PipeConfigRegionWritePlanEvent} in {@link ConfigRegionListeningQueue}. */ -public class PipeConfigPhysicalPlanTreePatternParseVisitor +public class PipeConfigTreePatternParseVisitor extends ConfigPhysicalPlanVisitor, UnionIoTDBTreePattern> { private static final Logger LOGGER = - LoggerFactory.getLogger(PipeConfigPhysicalPlanTreePatternParseVisitor.class); + LoggerFactory.getLogger(PipeConfigTreePatternParseVisitor.class); @Override public Optional visitPlan( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePrivilegeParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePrivilegeParseVisitor.java new file mode 100644 index 000000000000..f22f7c49accc --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePrivilegeParseVisitor.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.pipe.source; + +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.auth.entity.PrivilegeUnion; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.auth.AccessDeniedException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; +import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; +import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; +import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeactivateTemplatePlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteLogicalViewPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteTimeSeriesPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeUnsetSchemaTemplatePlan; +import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; +import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; +import org.apache.iotdb.confignode.consensus.request.write.template.ExtendSchemaTemplatePlan; +import org.apache.iotdb.confignode.service.ConfigNode; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE; + +public class PipeConfigTreePrivilegeParseVisitor + extends ConfigPhysicalPlanVisitor, String> { + private static final Logger LOGGER = + LoggerFactory.getLogger(PipeConfigTreePrivilegeParseVisitor.class); + private final boolean skip; + + PipeConfigTreePrivilegeParseVisitor(final boolean skip) { + this.skip = skip; + } + + @Override + public Optional visitPlan( + final ConfigPhysicalPlan plan, final String context) { + return Optional.of(plan); + } + + @Override + public Optional visitCreateDatabase( + final DatabaseSchemaPlan createDatabasePlan, final String userName) { + return canReadSysSchema(createDatabasePlan.getSchema().getName(), userName, true) + ? Optional.of(createDatabasePlan) + : Optional.empty(); + } + + @Override + public Optional visitAlterDatabase( + final DatabaseSchemaPlan alterDatabasePlan, final String userName) { + return canReadSysSchema(alterDatabasePlan.getSchema().getName(), userName, true) + ? Optional.of(alterDatabasePlan) + : Optional.empty(); + } + + @Override + public Optional visitDeleteDatabase( + final DeleteDatabasePlan deleteDatabasePlan, final String userName) { + return canReadSysSchema(deleteDatabasePlan.getName(), userName, true) + ? Optional.of(deleteDatabasePlan) + : Optional.empty(); + } + + @Override + public Optional visitCreateSchemaTemplate( + final CreateSchemaTemplatePlan createSchemaTemplatePlan, final String userName) { + return canShowSchemaTemplate(createSchemaTemplatePlan.getTemplate().getName(), userName) + ? Optional.of(createSchemaTemplatePlan) + : Optional.empty(); + } + + @Override + public Optional visitCommitSetSchemaTemplate( + final CommitSetSchemaTemplatePlan commitSetSchemaTemplatePlan, final String userName) { + return canReadSysSchema(commitSetSchemaTemplatePlan.getPath(), userName, false) + ? Optional.of(commitSetSchemaTemplatePlan) + : Optional.empty(); + } + + @Override + public Optional visitPipeUnsetSchemaTemplate( + final PipeUnsetSchemaTemplatePlan pipeUnsetSchemaTemplatePlan, final String userName) { + return canReadSysSchema(pipeUnsetSchemaTemplatePlan.getPath(), userName, false) + ? Optional.of(pipeUnsetSchemaTemplatePlan) + : Optional.empty(); + } + + @Override + public Optional visitExtendSchemaTemplate( + final ExtendSchemaTemplatePlan extendSchemaTemplatePlan, final String userName) { + return canShowSchemaTemplate( + extendSchemaTemplatePlan.getTemplateExtendInfo().getTemplateName(), userName) + ? Optional.of(extendSchemaTemplatePlan) + : Optional.empty(); + } + + public boolean canShowSchemaTemplate(final String templateName, final String userName) { + try { + return ConfigNode.getInstance() + .getConfigManager() + .getPermissionManager() + .checkUserPrivileges(userName, new PrivilegeUnion(PrivilegeType.SYSTEM)) + .getStatus() + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || ConfigNode.getInstance() + .getConfigManager() + .getClusterSchemaManager() + .getPathsSetTemplate(templateName, ALL_MATCH_SCOPE) + .getPathList() + .stream() + .anyMatch( + path -> { + try { + return ConfigNode.getInstance() + .getConfigManager() + .getPermissionManager() + .checkUserPrivileges( + userName, + new PrivilegeUnion( + Collections.singletonList( + new PartialPath(path) + .concatNode(MULTI_LEVEL_PATH_WILDCARD)), + PrivilegeType.READ_SCHEMA)) + .getStatus() + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } catch (final IllegalPathException e) { + throw new RuntimeException(e); + } + }); + } catch (final Exception e) { + LOGGER.warn( + "Un-parse-able path name encountered during template privilege trimming, please check", + e); + return false; + } + } + + public boolean canReadSysSchema( + final String path, final String userName, final boolean canSkipMulti) { + try { + return canSkipMulti + && ConfigNode.getInstance() + .getConfigManager() + .getPermissionManager() + .checkUserPrivileges( + userName, + new PrivilegeUnion( + Collections.singletonList(new PartialPath(path)), + PrivilegeType.READ_SCHEMA)) + .getStatus() + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || ConfigNode.getInstance() + .getConfigManager() + .getPermissionManager() + .checkUserPrivileges( + userName, + new PrivilegeUnion( + Collections.singletonList( + new PartialPath(path).concatNode(MULTI_LEVEL_PATH_WILDCARD)), + PrivilegeType.READ_SCHEMA)) + .getStatus() + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || ConfigNode.getInstance() + .getConfigManager() + .getPermissionManager() + .checkUserPrivileges(userName, new PrivilegeUnion(PrivilegeType.SYSTEM)) + .getStatus() + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } catch (final IllegalPathException e) { + LOGGER.warn("Un-parse-able path name encountered during privilege trimming, please check", e); + return false; + } + } + + @Override + public Optional visitGrantUser( + final AuthorTreePlan grantUserPlan, final String userName) { + return visitUserPlan(grantUserPlan, userName); + } + + @Override + public Optional visitRevokeUser( + final AuthorTreePlan revokeUserPlan, final String userName) { + return visitUserPlan(revokeUserPlan, userName); + } + + @Override + public Optional visitGrantRole( + final AuthorTreePlan revokeUserPlan, final String userName) { + return visitRolePlan(revokeUserPlan, userName); + } + + @Override + public Optional visitRevokeRole( + final AuthorTreePlan revokeUserPlan, final String userName) { + return visitRolePlan(revokeUserPlan, userName); + } + + private Optional visitUserPlan( + final AuthorTreePlan plan, final String userName) { + return ConfigNode.getInstance() + .getConfigManager() + .getPermissionManager() + .checkUserPrivileges(userName, new PrivilegeUnion(PrivilegeType.MANAGE_USER)) + .getStatus() + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? Optional.of(plan) + : Optional.empty(); + } + + private Optional visitRolePlan( + final AuthorTreePlan plan, final String userName) { + return ConfigNode.getInstance() + .getConfigManager() + .getPermissionManager() + .checkUserPrivileges(userName, new PrivilegeUnion(PrivilegeType.MANAGE_ROLE)) + .getStatus() + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? Optional.of(plan) + : Optional.empty(); + } + + @Override + public Optional visitPipeDeleteTimeSeries( + final PipeDeleteTimeSeriesPlan pipeDeleteTimeSeriesPlan, final String userName) { + try { + final PathPatternTree originalTree = + PathPatternTree.deserialize(pipeDeleteTimeSeriesPlan.getPatternTreeBytes()); + final PathPatternTree intersectedTree = + originalTree.intersectWithFullPathPrefixTree(getAuthorizedPTree(userName)); + if (!skip && !originalTree.equals(intersectedTree)) { + throw new AccessDeniedException( + "Not has privilege to transfer plan: " + pipeDeleteTimeSeriesPlan); + } + return !intersectedTree.isEmpty() + ? Optional.of(new PipeDeleteTimeSeriesPlan(intersectedTree.serialize())) + : Optional.empty(); + } catch (final IOException e) { + LOGGER.warn( + "Serialization failed for the delete time series plan in pipe transmission, skip transfer", + e); + return Optional.empty(); + } catch (final AuthException e) { + if (skip) { + return Optional.empty(); + } else { + throw new AccessDeniedException( + "Not has privilege to transfer plan: " + pipeDeleteTimeSeriesPlan); + } + } + } + + @Override + public Optional visitPipeDeleteLogicalView( + final PipeDeleteLogicalViewPlan pipeDeleteLogicalViewPlan, final String userName) { + try { + final PathPatternTree originalTree = + PathPatternTree.deserialize(pipeDeleteLogicalViewPlan.getPatternTreeBytes()); + final PathPatternTree intersectedTree = + originalTree.intersectWithFullPathPrefixTree(getAuthorizedPTree(userName)); + if (!skip && !originalTree.equals(intersectedTree)) { + throw new AccessDeniedException( + "Not has privilege to transfer plan: " + pipeDeleteLogicalViewPlan); + } + return !intersectedTree.isEmpty() + ? Optional.of(new PipeDeleteLogicalViewPlan(intersectedTree.serialize())) + : Optional.empty(); + } catch (final IOException e) { + LOGGER.warn( + "Serialization failed for the delete time series plan in pipe transmission, skip transfer", + e); + return Optional.empty(); + } catch (final AuthException e) { + if (skip) { + return Optional.empty(); + } else { + throw new AccessDeniedException( + "Not has privilege to transfer plan: " + pipeDeleteLogicalViewPlan); + } + } + } + + @Override + public Optional visitPipeDeactivateTemplate( + final PipeDeactivateTemplatePlan pipeDeactivateTemplatePlan, final String userName) { + try { + final Map> newTemplateSetInfo = new HashMap<>(); + for (final Map.Entry> templateEntry : + pipeDeactivateTemplatePlan.getTemplateSetInfo().entrySet()) { + for (final PartialPath intersectedPath : + getAllIntersectedPatterns( + templateEntry.getKey(), userName, pipeDeactivateTemplatePlan)) { + newTemplateSetInfo.put(intersectedPath, templateEntry.getValue()); + } + } + return !newTemplateSetInfo.isEmpty() + ? Optional.of(new PipeDeactivateTemplatePlan(newTemplateSetInfo)) + : Optional.empty(); + } catch (final AuthException e) { + if (skip) { + return Optional.empty(); + } else { + throw new AccessDeniedException( + "Not has privilege to transfer plan: " + pipeDeactivateTemplatePlan); + } + } + } + + @Override + public Optional visitTTL(final SetTTLPlan setTTLPlan, final String userName) { + try { + final List paths = + getAllIntersectedPatterns( + new PartialPath(setTTLPlan.getPathPattern()), userName, setTTLPlan); + // The intersectionList is either a singleton list or an empty list, because the pipe + // pattern and TTL path are each either a prefix path or a full path + return !paths.isEmpty() + ? Optional.of(new SetTTLPlan(paths.get(0).getNodes(), setTTLPlan.getTTL())) + : Optional.empty(); + } catch (final AuthException e) { + if (skip) { + return Optional.empty(); + } else { + throw new AccessDeniedException("Not has privilege to transfer plan: " + setTTLPlan); + } + } + } + + private List getAllIntersectedPatterns( + final PartialPath partialPath, final String userName, final ConfigPhysicalPlan plan) + throws AuthException { + final PathPatternTree thisPatternTree = new PathPatternTree(); + thisPatternTree.appendPathPattern(partialPath); + thisPatternTree.constructTree(); + final PathPatternTree intersectedTree = + thisPatternTree.intersectWithFullPathPrefixTree(getAuthorizedPTree(userName)); + if (!skip && !thisPatternTree.equals(intersectedTree)) { + throw new AccessDeniedException("Not has privilege to transfer plan: " + plan); + } + return intersectedTree.getAllPathPatterns(); + } + + private PathPatternTree getAuthorizedPTree(final String userName) throws AuthException { + return ConfigNode.getInstance() + .getConfigManager() + .getPermissionManager() + .fetchRawAuthorizedPTree(userName, PrivilegeType.READ_SCHEMA); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreeScopeParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreeScopeParseVisitor.java similarity index 98% rename from iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreeScopeParseVisitor.java rename to iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreeScopeParseVisitor.java index a1459c0f596b..d06f914b79e4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreeScopeParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreeScopeParseVisitor.java @@ -28,7 +28,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class PipeConfigPhysicalPlanTreeScopeParseVisitor +public class PipeConfigTreeScopeParseVisitor extends ConfigPhysicalPlanVisitor, Void> { @Override public Optional visitPlan(final ConfigPhysicalPlan plan, final Void context) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index bfe6e830cba9..a0986782c5be 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -33,6 +33,7 @@ import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; @@ -100,7 +101,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp; import org.apache.iotdb.consensus.exception.ConsensusException; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorInfo.java index 06bcd81b7fa0..743e0a3f09a5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorInfo.java @@ -24,9 +24,11 @@ import org.apache.iotdb.commons.auth.authorizer.BasicAuthorizer; import org.apache.iotdb.commons.auth.authorizer.IAuthorizer; import org.apache.iotdb.commons.auth.entity.ModelType; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.auth.entity.PrivilegeUnion; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.TestOnly; @@ -200,6 +202,11 @@ public TAuthizedPatternTreeResp generateAuthorizedPTree(String username, int per } } + public PathPatternTree generateRawAuthorizedPTree(final String username, final PrivilegeType type) + throws AuthException { + return authorPlanExecutor.generateRawAuthorizedPTree(username, type); + } + public TPermissionInfoResp checkRoleOfUser(String username, String roleName) { try { return authorPlanExecutor.checkRoleOfUser(username, roleName); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java index 9430cc06f023..239895b3a3cc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/AuthorPlanExecutor.java @@ -682,6 +682,33 @@ public TAuthizedPatternTreeResp generateAuthorizedPTree(String username, int per return resp; } + public PathPatternTree generateRawAuthorizedPTree(final String username, final PrivilegeType type) + throws AuthException { + final User user = authorizer.getUser(username); + final PathPatternTree pPtree = new PathPatternTree(); + if (user == null) { + return null; + } + + constructAuthorityScope(pPtree, user, type); + + for (final String roleName : user.getRoleSet()) { + Role role = authorizer.getRole(roleName); + if (role != null) { + constructAuthorityScope(pPtree, role, type); + } + } + pPtree.constructTree(); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + try { + pPtree.serialize(dataOutputStream); + } catch (final IOException e) { + return null; + } + return pPtree; + } + @Override public TPermissionInfoResp checkRoleOfUser(String username, String roleName) throws AuthException { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/IAuthorPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/IAuthorPlanExecutor.java index 9f93dba165b5..24f0d8ceb0a7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/IAuthorPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/auth/IAuthorPlanExecutor.java @@ -22,7 +22,9 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.commons.auth.entity.ModelType; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.auth.entity.PrivilegeUnion; +import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; @@ -54,6 +56,9 @@ public interface IAuthorPlanExecutor { TAuthizedPatternTreeResp generateAuthorizedPTree(String username, int permission) throws AuthException; + public PathPatternTree generateRawAuthorizedPTree(final String username, final PrivilegeType type) + throws AuthException; + TPermissionInfoResp checkRoleOfUser(String username, String roleName) throws AuthException; TPermissionInfoResp getUser(String username) throws AuthException; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNPhysicalPlanGenerator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNPhysicalPlanGenerator.java index 376fd29e1e7f..f36bd5fffbff 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNPhysicalPlanGenerator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/CNPhysicalPlanGenerator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.schema.node.role.IDatabaseMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.utils.AuthUtils; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils; @@ -43,7 +44,6 @@ import org.apache.iotdb.confignode.persistence.schema.mnode.factory.ConfigMNodeFactory; import org.apache.iotdb.confignode.persistence.schema.mnode.impl.ConfigTableNode; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.external.commons.io.IOUtils; import org.apache.tsfile.utils.Pair; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 1fc4b04c2b60..62fe57fb0b7c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -33,6 +33,7 @@ import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; @@ -96,7 +97,6 @@ import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.exception.sql.SemanticException; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; import org.apache.iotdb.rpc.RpcUtils; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeStore.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeStore.java index 3a7786ffa9b1..529c2aed5194 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeStore.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeStore.java @@ -26,13 +26,13 @@ import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.persistence.schema.mnode.IConfigMNode; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator.AbstractTraverserIterator; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator.MNodeIterator; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator.MemoryTraverserIterator; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; -import org.apache.iotdb.db.schemaengine.template.Template; import java.io.File; import java.util.Map; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplateTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplateTable.java index 99ce045d98e9..14d31158d765 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplateTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplateTable.java @@ -20,9 +20,9 @@ package org.apache.iotdb.confignode.persistence.schema; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.exception.metadata.template.UndefinedTemplateException; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; import org.apache.tsfile.common.conf.TSFileDescriptor; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedure.java index 1df9da1509dc..bd3e623ec9d8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedure.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; @@ -39,7 +40,6 @@ import org.apache.iotdb.confignode.procedure.state.schema.DeactivateTemplateState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.consensus.exception.ConsensusException; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.mpp.rpc.thrift.TConstructSchemaBlackListWithTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TDeactivateTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDeleteSchemaReq; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java index caa257faab8e..e2ab08e93666 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCType; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; @@ -37,7 +38,6 @@ import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateResp; import org.apache.iotdb.mpp.rpc.thrift.TCountPathsUsingTemplateReq; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTemplateProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTemplateProcedure.java index a5463439f66c..5509037c6fa9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTemplateProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTemplateProcedure.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; @@ -45,7 +46,6 @@ import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.exception.metadata.template.TemplateIncompatibleException; import org.apache.iotdb.db.exception.metadata.template.UndefinedTemplateException; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/UnsetTemplateProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/UnsetTemplateProcedure.java index 464c6b7c21b0..793e6fa6b181 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/UnsetTemplateProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/UnsetTemplateProcedure.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; @@ -35,7 +36,6 @@ import org.apache.iotdb.confignode.procedure.state.schema.UnsetTemplateState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.db.exception.metadata.template.TemplateIsInUseException; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; import org.apache.iotdb.mpp.rpc.thrift.TUpdateTemplateReq; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 4381c120cb44..2b5d527d75da 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -53,6 +53,7 @@ import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; @@ -174,7 +175,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.confignode.rpc.thrift.TPipeSinkInfo; import org.apache.iotdb.confignode.rpc.thrift.TTriggerState; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; import org.apache.iotdb.trigger.api.enums.FailureStrategy; import org.apache.iotdb.trigger.api.enums.TriggerEvent; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanScopeParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigScopeParseVisitorTest.java similarity index 98% rename from iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanScopeParseVisitorTest.java rename to iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigScopeParseVisitorTest.java index 1a0ca3541eac..67887d58a073 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanScopeParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigScopeParseVisitorTest.java @@ -33,7 +33,7 @@ import java.util.HashSet; import java.util.stream.Collectors; -public class PipeConfigPhysicalPlanScopeParseVisitorTest { +public class PipeConfigScopeParseVisitorTest { @Test public void testTreeScopeParsing() { testTreeScopeParsing(ConfigPhysicalPlanType.GrantRole, false); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java similarity index 99% rename from iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java rename to iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java index 10bc29012728..fea613bc47bb 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java @@ -52,7 +52,7 @@ import java.util.ArrayList; import java.util.Collections; -public class PipeConfigPhysicalPlanTablePatternParseVisitorTest { +public class PipeConfigTablePatternParseVisitorTest { private final TablePattern tablePattern = new TablePattern(true, "^db[0-9]", "a.*b"); @Test diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitorTest.java similarity index 99% rename from iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java rename to iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitorTest.java index 442351cc19d2..0502879cbcf6 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitorTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; @@ -37,7 +38,6 @@ import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.ExtendSchemaTemplatePlan; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; import org.apache.tsfile.enums.TSDataType; @@ -53,7 +53,7 @@ import java.util.HashSet; import java.util.List; -public class PipeConfigPhysicalPlanTreePatternParseVisitorTest { +public class PipeConfigTreePatternParseVisitorTest { private final UnionIoTDBTreePattern prefixPathPattern = new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CNPhysicalPlanGeneratorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CNPhysicalPlanGeneratorTest.java index b9d453d317ce..b1b5f569f42a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CNPhysicalPlanGeneratorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CNPhysicalPlanGeneratorTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.utils.AuthUtils; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; @@ -39,7 +40,6 @@ import org.apache.iotdb.confignode.persistence.schema.CNSnapshotFileType; import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfoTest.java index 0d9bdf26a982..1cf0b2913a83 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfoTest.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; @@ -34,7 +35,6 @@ import org.apache.iotdb.confignode.consensus.response.template.TemplateInfoResp; import org.apache.iotdb.confignode.consensus.response.template.TemplateSetInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; import org.apache.tsfile.enums.TSDataType; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/TemplateTableTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/TemplateTableTest.java index 72c96d096c01..dd5bb5d8bb8b 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/TemplateTableTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/TemplateTableTest.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; -import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.io.FileUtils; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/receiver/PipeEnrichedProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/receiver/PipeEnrichedProcedureTest.java index 9a3054cacda9..856d3687bd2d 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/receiver/PipeEnrichedProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/receiver/PipeEnrichedProcedureTest.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.schema.view.viewExpression.leaf.ConstantViewOperand; import org.apache.iotdb.commons.trigger.TriggerInformation; @@ -63,7 +64,6 @@ import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.confignode.rpc.thrift.TTriggerState; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.trigger.api.enums.FailureStrategy; import org.apache.iotdb.trigger.api.enums.TriggerEvent; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedureTest.java index 87efb7a93746..a872b3a63d1a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedureTest.java @@ -21,8 +21,8 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.procedure.store.ProcedureType; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/UnsetTemplateProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/UnsetTemplateProcedureTest.java index b288418e1b56..be9900ee90ca 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/UnsetTemplateProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/UnsetTemplateProcedureTest.java @@ -21,8 +21,8 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.confignode.procedure.store.ProcedureType; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 6b9ba0a26120..3f05e78066a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.exception.metadata.MeasurementAlreadyExistException; import org.apache.iotdb.db.exception.metadata.template.TemplateIsInUseException; @@ -73,7 +74,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateAlignedTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java index f979d4763b5d..c5a5cb4948d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.agent.task.connection; import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; @@ -32,6 +33,7 @@ import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; +import org.apache.iotdb.db.pipe.source.schemaregion.PipePlanTreePrivilegeParseVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.pipe.api.collector.EventCollector; @@ -115,7 +117,8 @@ private void parseAndCollectEvent(final PipeInsertNodeTabletInsertionEvent sourc } } - private void parseAndCollectEvent(final PipeRawTabletInsertionEvent sourceEvent) { + private void parseAndCollectEvent(final PipeRawTabletInsertionEvent sourceEvent) + throws IllegalPathException { if (sourceEvent.shouldParseTimeOrPattern()) { collectParsedRawTableEvent(sourceEvent.parseEventWithPatternOrTime()); } else { @@ -170,9 +173,20 @@ private void parseAndCollectEvent(final PipeDeleteDataNodeEvent deleteDataEvent) // Only used by events containing delete data node, no need to bind progress index here since // delete data event does not have progress index currently (deleteDataEvent.getDeleteDataNode() instanceof DeleteDataNode - ? IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR.process( - deleteDataEvent.getDeleteDataNode(), - (UnionIoTDBTreePattern) deleteDataEvent.getTreePattern()) + ? IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .process( + deleteDataEvent.getDeleteDataNode(), + (UnionIoTDBTreePattern) deleteDataEvent.getTreePattern()) + .flatMap( + planNode -> + new PipePlanTreePrivilegeParseVisitor( + deleteDataEvent.isSkipIfNoPrivileges()) + .process( + planNode, + new UserEntity( + Long.parseLong(deleteDataEvent.getUserId()), + deleteDataEvent.getUserName(), + deleteDataEvent.getCliHostname()))) : IoTDBSchemaRegionSource.TABLE_PATTERN_PARSE_VISITOR .process(deleteDataEvent.getDeleteDataNode(), deleteDataEvent.getTablePattern()) .flatMap( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/PipeInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/PipeInsertionEvent.java index 2641f522654d..ce491b92ef46 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/PipeInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/PipeInsertionEvent.java @@ -56,6 +56,7 @@ public abstract class PipeInsertionEvent extends EnrichedEvent { protected String treeModelDatabaseName; // lazy initialization protected String tableModelDatabaseName; // lazy initialization + protected boolean shouldParse4Privilege = false; protected PipeInsertionEvent( final String pipeName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java index 63cae75fbbd1..b20d6d3655cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java @@ -19,10 +19,14 @@ package org.apache.iotdb.db.pipe.event.common.tablet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; +import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; @@ -54,7 +58,9 @@ import org.apache.iotdb.pipe.api.collector.RowCollector; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -267,24 +273,25 @@ public boolean isGeneratedByPipe() { } @Override - public void throwIfNoPrivilege() { - if (skipIfNoPrivileges || !isTableModelEvent()) { + public void throwIfNoPrivilege() throws Exception { + if (skipIfNoPrivileges) { return; } if (Objects.nonNull(insertNode.getTargetPath())) { - checkTableName( - DeviceIDFactory.getInstance().getDeviceID(insertNode.getTargetPath()).getTableName()); + if (isTableModelEvent()) { + checkTableName( + DeviceIDFactory.getInstance().getDeviceID(insertNode.getTargetPath()).getTableName()); + } else { + checkTreePattern(insertNode.getDeviceID(), insertNode.getMeasurements()); + } } else if (insertNode instanceof InsertRowsNode) { - for (final String tableName : - ((InsertRowsNode) insertNode) - .getInsertRowNodeList().stream() - .map( - node -> - DeviceIDFactory.getInstance() - .getDeviceID(node.getTargetPath()) - .getTableName()) - .collect(Collectors.toSet())) { - checkTableName(tableName); + for (final InsertNode node : ((InsertRowsNode) insertNode).getInsertRowNodeList()) { + if (isTableModelEvent()) { + checkTableName( + DeviceIDFactory.getInstance().getDeviceID(node.getTargetPath()).getTableName()); + } else { + checkTreePattern(node.getDeviceID(), node.getMeasurements()); + } } } } @@ -302,6 +309,29 @@ private void checkTableName(final String tableName) { } } + private void checkTreePattern(final IDeviceID deviceID, final String[] measurements) + throws IllegalPathException { + final List measurementList = new ArrayList<>(); + for (final String measurement : measurements) { + if (!treePattern.matchesMeasurement(deviceID, measurement)) { + measurementList.add(new MeasurementPath(deviceID, measurement)); + } + } + final TSStatus status = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilege4Pipe( + new UserEntity(Long.parseLong(userId), userName, cliHostname), + measurementList, + PrivilegeType.READ_DATA); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != status.getCode()) { + if (skipIfNoPrivileges) { + shouldParse4Privilege = true; + } else { + throw new AccessDeniedException(status.getMessage()); + } + } + } + @Override public boolean mayEventTimeOverlappedWithTimeRange() { try { @@ -447,13 +477,26 @@ private List initEventParsers() { case INSERT_ROW: case INSERT_TABLET: eventParsers.add( - new TabletInsertionEventTreePatternParser(pipeTaskMeta, this, node, treePattern)); + new TabletInsertionEventTreePatternParser( + pipeTaskMeta, + this, + node, + treePattern, + shouldParse4Privilege + ? new UserEntity(Long.parseLong(userId), userName, cliHostname) + : null)); break; case INSERT_ROWS: for (final InsertRowNode insertRowNode : ((InsertRowsNode) node).getInsertRowNodeList()) { eventParsers.add( new TabletInsertionEventTreePatternParser( - pipeTaskMeta, this, insertRowNode, treePattern)); + pipeTaskMeta, + this, + insertRowNode, + treePattern, + shouldParse4Privilege + ? new UserEntity(Long.parseLong(userId), userName, cliHostname) + : null)); } break; case RELATIONAL_INSERT_ROW: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java index adcef5128f52..f23e19f4c0e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java @@ -19,8 +19,10 @@ package org.apache.iotdb.db.pipe.event.common.tablet; +import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; @@ -406,13 +408,21 @@ public EnrichedEvent getSourceEvent() { @Override public Iterable processRowByRow( final BiConsumer consumer) { - return initEventParser().processRowByRow(consumer); + try { + return initEventParser().processRowByRow(consumer); + } catch (final Exception e) { + throw new RuntimeException(e); + } } @Override public Iterable processTablet( final BiConsumer consumer) { - return initEventParser().processTablet(consumer); + try { + return initEventParser().processTablet(consumer); + } catch (final Exception e) { + throw new RuntimeException(e); + } } /////////////////////////// convertToTablet /////////////////////////// @@ -435,21 +445,28 @@ private TabletInsertionEventParser initEventParser() { eventParser = tablet.getDeviceId().startsWith("root.") ? new TabletInsertionEventTreePatternParser( - pipeTaskMeta, this, tablet, isAligned, treePattern) + pipeTaskMeta, + this, + tablet, + isAligned, + treePattern, + shouldParse4Privilege + ? new UserEntity(Long.parseLong(userId), userName, cliHostname) + : null) : new TabletInsertionEventTablePatternParser( pipeTaskMeta, this, tablet, isAligned, tablePattern); } return eventParser; } - public long count() { + public long count() throws IllegalPathException { final Tablet convertedTablet = shouldParseTimeOrPattern() ? convertToTablet() : tablet; return (long) convertedTablet.getRowSize() * convertedTablet.getSchemas().size(); } /////////////////////////// parsePatternOrTime /////////////////////////// - public PipeRawTabletInsertionEvent parseEventWithPatternOrTime() { + public PipeRawTabletInsertionEvent parseEventWithPatternOrTime() throws IllegalPathException { return new PipeRawTabletInsertionEvent( getRawIsTableModelEvent(), getSourceDatabaseNameFromDataRegion(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java index e10051325b31..4e3286546328 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.tablet.parser; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; @@ -189,7 +190,7 @@ protected void parse(final InsertRowNode insertRowNode) { } } - protected void parse(final InsertTabletNode insertTabletNode) { + protected void parse(final InsertTabletNode insertTabletNode) throws IllegalPathException { final int originColumnSize = insertTabletNode.getMeasurements().length; final Integer[] originColumnIndex2FilteredColumnIndexMapperList = new Integer[originColumnSize]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java index de68404fb2a3..76c0507daf2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.tablet.parser; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; @@ -51,7 +52,8 @@ public TabletInsertionEventTablePatternParser( final PipeTaskMeta pipeTaskMeta, final EnrichedEvent sourceEvent, final InsertNode insertNode, - final TablePattern pattern) { + final TablePattern pattern) + throws IllegalPathException { super(pipeTaskMeta, sourceEvent); this.pattern = pattern; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreePatternParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreePatternParser.java index 68fb0e50b95f..44eebb10e172 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreePatternParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreePatternParser.java @@ -19,10 +19,15 @@ package org.apache.iotdb.db.pipe.event.common.tablet.parser; +import org.apache.iotdb.commons.audit.IAuditEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.pipe.event.common.row.PipeRow; import org.apache.iotdb.db.pipe.event.common.row.PipeRowCollector; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; @@ -31,6 +36,7 @@ import org.apache.iotdb.pipe.api.access.Row; import org.apache.iotdb.pipe.api.collector.RowCollector; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; @@ -44,14 +50,18 @@ public class TabletInsertionEventTreePatternParser extends TabletInsertionEventParser { private final TreePattern pattern; + private final IAuditEntity entity; public TabletInsertionEventTreePatternParser( final PipeTaskMeta pipeTaskMeta, final EnrichedEvent sourceEvent, final InsertNode insertNode, - final TreePattern pattern) { + final TreePattern pattern, + final IAuditEntity entity) + throws IllegalPathException { super(pipeTaskMeta, sourceEvent); this.pattern = pattern; + this.entity = entity; if (insertNode instanceof InsertRowNode) { parse((InsertRowNode) insertNode); @@ -68,17 +78,19 @@ public TabletInsertionEventTreePatternParser( final EnrichedEvent sourceEvent, final Tablet tablet, final boolean isAligned, - final TreePattern pattern) { + final TreePattern pattern, + final IAuditEntity entity) { super(pipeTaskMeta, sourceEvent); this.pattern = pattern; + this.entity = entity; parse(tablet, isAligned); } @TestOnly public TabletInsertionEventTreePatternParser( - final InsertNode insertNode, final TreePattern pattern) { - this(null, null, insertNode, pattern); + final InsertNode insertNode, final TreePattern pattern) throws IllegalPathException { + this(null, null, insertNode, pattern, null); } @Override @@ -94,7 +106,8 @@ protected void generateColumnIndexMapper( // case 1: for example, pattern is root.a.b or pattern is null and device is root.a.b.c // in this case, all data can be matched without checking the measurements - if (Objects.isNull(pattern) || pattern.isRoot() || pattern.coversDevice(deviceId)) { + if (Objects.isNull(entity) + && (Objects.isNull(pattern) || pattern.isRoot() || pattern.coversDevice(deviceId))) { for (int i = 0; i < originColumnSize; i++) { originColumnIndex2FilteredColumnIndexMapperList[i] = i; } @@ -113,8 +126,20 @@ else if (pattern.mayOverlapWithDevice(deviceId)) { continue; } - if (pattern.matchesMeasurement(deviceId, measurement)) { - originColumnIndex2FilteredColumnIndexMapperList[i] = filteredCount++; + try { + if (pattern.matchesMeasurement(deviceId, measurement) + && (Objects.isNull(entity) + || AuthorityChecker.getAccessControl() + .checkSeriesPrivilege4Pipe( + entity, + Collections.singletonList(new MeasurementPath(deviceId, measurement)), + PrivilegeType.READ_DATA) + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode())) { + originColumnIndex2FilteredColumnIndexMapperList[i] = filteredCount++; + } + } catch (final IllegalPathException e) { + throw new RuntimeException(e); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index 2a1ab3f5a35f..618e68dc7863 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -19,11 +19,16 @@ package org.apache.iotdb.db.pipe.event.common.tsfile; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeOutOfMemoryCriticalException; +import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; @@ -47,6 +52,7 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.file.metadata.IDeviceID; import org.slf4j.Logger; @@ -54,8 +60,10 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -77,7 +85,6 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent private boolean isWithMod; private File modFile; private final File sharedModFile; - private boolean shouldParse4Privilege = false; protected final boolean isLoaded; protected final boolean isGeneratedByPipe; @@ -92,6 +99,7 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent protected volatile ProgressIndex overridingProgressIndex; private Set tableNames; + private Map treeSchemaMap; public PipeTsFileInsertionEvent( final Boolean isTableModelEvent, @@ -441,33 +449,86 @@ public boolean isGeneratedByPipe() { @Override public void throwIfNoPrivilege() { try { - if (!isTableModelEvent() || AuthorityChecker.SUPER_USER.equals(userName)) { + if (AuthorityChecker.SUPER_USER.equals(userName)) { return; } if (!waitForTsFileClose()) { LOGGER.info("Temporary tsFile {} detected, will skip its transfer.", tsFile); return; } - for (final String table : tableNames) { - if (!tablePattern.matchesDatabase(getTableModelDatabaseName()) - || !tablePattern.matchesTable(table)) { - continue; + if (isTableModelEvent()) { + for (final String table : tableNames) { + if (!tablePattern.matchesDatabase(getTableModelDatabaseName()) + || !tablePattern.matchesTable(table)) { + continue; + } + if (!AuthorityChecker.getAccessControl() + .checkCanSelectFromTable4Pipe( + userName, + new QualifiedObjectName(getTableModelDatabaseName(), table), + new UserEntity(Long.parseLong(userId), userName, cliHostname))) { + if (skipIfNoPrivileges) { + shouldParse4Privilege = true; + } else { + throw new AccessDeniedException( + String.format( + "No privilege for SELECT for user %s at table %s.%s", + userName, tableModelDatabaseName, table)); + } + } } - if (!AuthorityChecker.getAccessControl() - .checkCanSelectFromTable4Pipe( - userName, - new QualifiedObjectName(getTableModelDatabaseName(), table), - new UserEntity(Long.parseLong(userId), userName, cliHostname))) { + } + // Real-time tsFiles + else if (Objects.nonNull(treeSchemaMap)) { + final List measurementList = new ArrayList<>(); + for (final Map.Entry entry : treeSchemaMap.entrySet()) { + final IDeviceID deviceID = entry.getKey(); + for (final String measurement : entry.getValue()) { + if (treePattern.matchesMeasurement(deviceID, measurement)) { + measurementList.add(new MeasurementPath(deviceID, measurement)); + } + } + } + final TSStatus status = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilege4Pipe( + new UserEntity(Long.parseLong(userId), userName, cliHostname), + measurementList, + PrivilegeType.READ_DATA); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != status.getCode()) { if (skipIfNoPrivileges) { shouldParse4Privilege = true; } else { - throw new AccessDeniedException( - String.format( - "No privilege for SELECT for user %s at table %s.%s", - userName, tableModelDatabaseName, table)); + throw new AccessDeniedException(status.getMessage()); } } } + // Historical tsFiles + // Coarse filter, will be judged in inner class + else { + final Set devices = getDeviceSet(); + if (Objects.nonNull(devices)) { + final List measurementList = new ArrayList<>(); + for (final IDeviceID device : devices) { + measurementList.add(new MeasurementPath(device, IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); + } + final TSStatus status = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilege4Pipe( + new UserEntity(Long.parseLong(userId), userName, cliHostname), + measurementList, + PrivilegeType.READ_DATA); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != status.getCode()) { + if (skipIfNoPrivileges) { + shouldParse4Privilege = true; + } else { + throw new AccessDeniedException(status.getMessage()); + } + } + } else { + shouldParse4Privilege = true; + } + } } catch (final AccessDeniedException e) { throw e; } catch (final Exception e) { @@ -485,6 +546,10 @@ public void throwIfNoPrivilege() { resource.getTsFilePath(), e.getMessage()); LOGGER.warn(errorMsg, e); throw new PipeException(errorMsg, e); + } finally { + // GC useless + tableNames = null; + treeSchemaMap = null; } } @@ -524,13 +589,21 @@ private Set getDeviceSet() throws IOException { if (Objects.nonNull(deviceIsAlignedMap)) { return deviceIsAlignedMap.keySet(); } - return resource.getDevices(); + try { + return resource.getDevices(); + } catch (final Exception e) { + return null; + } } public void setTableNames(final Set tableNames) { this.tableNames = tableNames; } + public void setTreeSchemaMap(final Map treeSchemaMap) { + this.treeSchemaMap = treeSchemaMap; + } + /////////////////////////// PipeInsertionEvent /////////////////////////// @Override @@ -548,11 +621,11 @@ public boolean isTableModelEvent() { @FunctionalInterface public interface TabletInsertionEventConsumer { - void consume(final PipeRawTabletInsertionEvent event); + void consume(final PipeRawTabletInsertionEvent event) throws IllegalPathException; } public void consumeTabletInsertionEventsWithRetry( - final TabletInsertionEventConsumer consumer, final String callerName) throws PipeException { + final TabletInsertionEventConsumer consumer, final String callerName) throws Exception { final Iterable iterable = toTabletInsertionEvents(); final Iterator iterator = iterable.iterator(); int tabletEventCount = 0; @@ -604,10 +677,6 @@ public Iterable toTabletInsertionEvents(final long timeout "Pipe skipping temporary TsFile's parsing which shouldn't be transferred: {}", tsFile); return Collections.emptyList(); } - // Skip if is table events and tree model - if (Objects.isNull(userName) && isTableModelEvent()) { - return Collections.emptyList(); - } waitForResourceEnough4Parsing(timeoutMs); return initEventParser().toTabletInsertionEvents(); } catch (final Exception e) { @@ -698,11 +767,13 @@ private TsFileInsertionEventParser initEventParser() { pipeTaskMeta, // Do not parse privilege if it should not be parsed // To avoid renaming of the tsFile database - shouldParse4Privilege ? userName : null, + shouldParse4Privilege + ? new UserEntity(Long.parseLong(userId), userName, cliHostname) + : null, this) .provide(isWithMod)); return eventParser.get(); - } catch (final IOException e) { + } catch (final Exception e) { close(); final String errorMsg = String.format("Read TsFile %s error.", tsFile.getPath()); @@ -711,7 +782,7 @@ private TsFileInsertionEventParser initEventParser() { } } - public long count(final boolean skipReportOnCommit) throws IOException { + public long count(final boolean skipReportOnCommit) throws Exception { AtomicLong count = new AtomicLong(); if (shouldParseTime()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java index 04b2a3bfd81c..82f3e0ea2b84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser; +import org.apache.iotdb.commons.audit.IAuditEntity; import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; @@ -48,6 +49,8 @@ public abstract class TsFileInsertionEventParser implements AutoCloseable { protected final String pipeName; protected final long creationTime; + protected IAuditEntity entity; + protected boolean skipIfNoPrivileges; protected final TreePattern treePattern; // used to filter data protected final TablePattern tablePattern; // used to filter data @@ -80,9 +83,13 @@ protected TsFileInsertionEventParser( final long startTime, final long endTime, final PipeTaskMeta pipeTaskMeta, + final IAuditEntity entity, + final boolean skipIfNoPrivileges, final PipeInsertionEvent sourceEvent) { this.pipeName = pipeName; this.creationTime = creationTime; + this.entity = entity; + this.skipIfNoPrivileges = skipIfNoPrivileges; this.treePattern = treePattern; this.tablePattern = tablePattern; @@ -159,7 +166,6 @@ protected void recordTabletMetrics(final Tablet tablet) { @Override public void close() { - tabletInsertionIterable = null; // Time recording is now handled in Iterator.hasNext(), no need to record here diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java index 8afd16277909..d9d4e6aed9f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser; +import org.apache.iotdb.commons.audit.IAuditEntity; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; @@ -53,7 +55,7 @@ public class TsFileInsertionEventParserProvider { protected final PipeTaskMeta pipeTaskMeta; protected final PipeTsFileInsertionEvent sourceEvent; - private final String userName; + private final IAuditEntity entity; public TsFileInsertionEventParserProvider( final String pipeName, @@ -64,7 +66,7 @@ public TsFileInsertionEventParserProvider( final long startTime, final long endTime, final PipeTaskMeta pipeTaskMeta, - final String userName, + final IAuditEntity entity, final PipeTsFileInsertionEvent sourceEvent) { this.pipeName = pipeName; this.creationTime = creationTime; @@ -74,11 +76,12 @@ public TsFileInsertionEventParserProvider( this.startTime = startTime; this.endTime = endTime; this.pipeTaskMeta = pipeTaskMeta; - this.userName = userName; + this.entity = entity; this.sourceEvent = sourceEvent; } - public TsFileInsertionEventParser provide(final boolean isWithMod) throws IOException { + public TsFileInsertionEventParser provide(final boolean isWithMod) + throws IOException, IllegalPathException { if (pipeName != null) { PipeTsFileToTabletsMetrics.getInstance() .markTsFileToTabletInvocation(pipeName + "_" + creationTime); @@ -93,7 +96,7 @@ public TsFileInsertionEventParser provide(final boolean isWithMod) throws IOExce startTime, endTime, pipeTaskMeta, - userName, + entity, sourceEvent, isWithMod); } @@ -110,6 +113,8 @@ public TsFileInsertionEventParser provide(final boolean isWithMod) throws IOExce startTime, endTime, pipeTaskMeta, + entity, + sourceEvent.isSkipIfNoPrivileges(), sourceEvent, isWithMod); } @@ -131,7 +136,10 @@ public TsFileInsertionEventParser provide(final boolean isWithMod) throws IOExce endTime, pipeTaskMeta, sourceEvent, - isWithMod); + entity, + sourceEvent.isSkipIfNoPrivileges(), + null, + false); } final Map deviceIsAlignedMap = @@ -147,6 +155,8 @@ public TsFileInsertionEventParser provide(final boolean isWithMod) throws IOExce startTime, endTime, pipeTaskMeta, + entity, + sourceEvent.isSkipIfNoPrivileges(), sourceEvent, isWithMod); } @@ -165,6 +175,8 @@ public TsFileInsertionEventParser provide(final boolean isWithMod) throws IOExce startTime, endTime, pipeTaskMeta, + entity, + sourceEvent.isSkipIfNoPrivileges(), sourceEvent, isWithMod) : new TsFileInsertionEventQueryParser( @@ -176,6 +188,8 @@ public TsFileInsertionEventParser provide(final boolean isWithMod) throws IOExce endTime, pipeTaskMeta, sourceEvent, + entity, + sourceEvent.isSkipIfNoPrivileges(), filteredDeviceIsAlignedMap, isWithMod); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index 22f1cffdd0dd..cc497635aab3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -19,10 +19,17 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.query; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.IAuditEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.auth.AccessDeniedException; +import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.pipe.event.common.PipeInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.TsFileInsertionEventParser; @@ -34,6 +41,7 @@ import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; @@ -49,6 +57,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -77,7 +86,7 @@ public TsFileInsertionEventQueryParser( final long startTime, final long endTime, final PipeInsertionEvent sourceEvent) - throws IOException { + throws IOException, IllegalPathException { this(null, 0, tsFile, pattern, startTime, endTime, null, sourceEvent, false); } @@ -91,7 +100,7 @@ public TsFileInsertionEventQueryParser( final PipeTaskMeta pipeTaskMeta, final PipeInsertionEvent sourceEvent, final boolean isWithMod) - throws IOException { + throws IOException, IllegalPathException { this( pipeName, creationTime, @@ -102,6 +111,8 @@ public TsFileInsertionEventQueryParser( pipeTaskMeta, sourceEvent, null, + false, + null, isWithMod); } @@ -114,10 +125,22 @@ public TsFileInsertionEventQueryParser( final long endTime, final PipeTaskMeta pipeTaskMeta, final PipeInsertionEvent sourceEvent, + final IAuditEntity entity, + final boolean skipIfNoPrivileges, final Map deviceIsAlignedMap, final boolean isWithMod) - throws IOException { - super(pipeName, creationTime, pattern, null, startTime, endTime, pipeTaskMeta, sourceEvent); + throws IOException, IllegalPathException { + super( + pipeName, + creationTime, + pattern, + null, + startTime, + endTime, + pipeTaskMeta, + entity, + skipIfNoPrivileges, + sourceEvent); try { currentModifications = @@ -239,7 +262,8 @@ public TsFileInsertionEventQueryParser( } private Map> filterDeviceMeasurementsMapByPattern( - final Map> originalDeviceMeasurementsMap) { + final Map> originalDeviceMeasurementsMap) + throws IllegalPathException { final Map> filteredDeviceMeasurementsMap = new HashMap<>(); for (Map.Entry> entry : originalDeviceMeasurementsMap.entrySet()) { final IDeviceID deviceId = entry.getKey(); @@ -261,6 +285,20 @@ else if (treePattern.mayOverlapWithDevice(deviceId)) { for (final String measurement : entry.getValue()) { if (treePattern.matchesMeasurement(deviceId, measurement)) { + if (Objects.nonNull(entity)) { + final TSStatus status = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilege4Pipe( + entity, + Collections.singletonList(new MeasurementPath(deviceId, measurement)), + PrivilegeType.READ_DATA); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (skipIfNoPrivileges) { + continue; + } + throw new AccessDeniedException(status.getMessage()); + } + } filteredMeasurements.add(measurement); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index 80382bf82b52..2c1cc27b80a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -19,9 +19,16 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.scan; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.audit.IAuditEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.auth.AccessDeniedException; +import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.pipe.event.common.PipeInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.TsFileInsertionEventParser; @@ -32,6 +39,7 @@ import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.constant.TsFileConstant; @@ -103,10 +111,22 @@ public TsFileInsertionEventScanParser( final long startTime, final long endTime, final PipeTaskMeta pipeTaskMeta, + final IAuditEntity entity, + final boolean skipIfNoPrivileges, final PipeInsertionEvent sourceEvent, final boolean isWithMod) - throws IOException { - super(pipeName, creationTime, pattern, null, startTime, endTime, pipeTaskMeta, sourceEvent); + throws IOException, IllegalPathException { + super( + pipeName, + creationTime, + pattern, + null, + startTime, + endTime, + pipeTaskMeta, + entity, + skipIfNoPrivileges, + sourceEvent); this.startTime = startTime; this.endTime = endTime; @@ -151,8 +171,19 @@ public TsFileInsertionEventScanParser( final PipeTaskMeta pipeTaskMeta, final PipeInsertionEvent sourceEvent, final boolean isWithMod) - throws IOException { - this(null, 0, tsFile, pattern, startTime, endTime, pipeTaskMeta, sourceEvent, isWithMod); + throws IOException, IllegalPathException { + this( + null, + 0, + tsFile, + pattern, + startTime, + endTime, + pipeTaskMeta, + null, + false, + sourceEvent, + isWithMod); } @Override @@ -326,7 +357,7 @@ private Tablet getNextTablet() { } } - private void prepareData() throws IOException { + private void prepareData() throws IOException, IllegalPathException { do { do { moveToNextChunkReader(); @@ -428,7 +459,8 @@ private boolean putValueToColumns(final BatchData data, final Tablet tablet, fin return isNeedFillTime; } - private void moveToNextChunkReader() throws IOException, IllegalStateException { + private void moveToNextChunkReader() + throws IOException, IllegalStateException, IllegalPathException { ChunkHeader chunkHeader; long valueChunkSize = 0; final List valueChunkList = new ArrayList<>(); @@ -497,6 +529,22 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { } } + if (Objects.nonNull(entity)) { + final TSStatus status = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilege4Pipe( + entity, + Collections.singletonList( + new MeasurementPath(currentDevice, chunkHeader.getMeasurementID())), + PrivilegeType.READ_DATA); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (skipIfNoPrivileges) { + continue; + } + throw new AccessDeniedException(status.getMessage()); + } + } + if (chunkHeader.getDataSize() > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { PipeDataNodeResourceManager.memory() .forceResize(allocatedMemoryBlockForChunk, chunkHeader.getDataSize()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java index 8ec8106a4963..8ac2d4e9b71d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.table; -import org.apache.iotdb.commons.audit.UserEntity; +import org.apache.iotdb.commons.audit.IAuditEntity; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; @@ -50,7 +50,6 @@ public class TsFileInsertionEventTableParser extends TsFileInsertionEventParser private final long startTime; private final long endTime; private final TablePattern tablePattern; - private final String userName; private final boolean isWithMod; private final PipeMemoryBlock allocatedMemoryBlockForBatchData; @@ -66,11 +65,21 @@ public TsFileInsertionEventTableParser( final long startTime, final long endTime, final PipeTaskMeta pipeTaskMeta, - final String userName, + final IAuditEntity entity, final PipeInsertionEvent sourceEvent, final boolean isWithMod) throws IOException { - super(pipeName, creationTime, null, pattern, startTime, endTime, pipeTaskMeta, sourceEvent); + super( + pipeName, + creationTime, + null, + pattern, + startTime, + endTime, + pipeTaskMeta, + entity, + true, + sourceEvent); this.isWithMod = isWithMod; try { @@ -103,7 +112,7 @@ public TsFileInsertionEventTableParser( this.endTime = endTime; this.tablePattern = pattern; - this.userName = userName; + this.entity = entity; tsFileSequenceReader = new TsFileSequenceReader(tsFile.getPath(), true, true); } catch (final Exception e) { close(); @@ -117,21 +126,12 @@ public TsFileInsertionEventTableParser( final long startTime, final long endTime, final PipeTaskMeta pipeTaskMeta, - final String userName, + final IAuditEntity entity, final PipeInsertionEvent sourceEvent, final boolean isWithMod) throws IOException { this( - null, - 0, - tsFile, - pattern, - startTime, - endTime, - pipeTaskMeta, - userName, - sourceEvent, - isWithMod); + null, 0, tsFile, pattern, startTime, endTime, pipeTaskMeta, entity, sourceEvent, isWithMod); } @Override @@ -182,15 +182,15 @@ && hasTablePrivilege(entry.getKey()), } private boolean hasTablePrivilege(final String tableName) { - return Objects.isNull(userName) + return Objects.isNull(entity) || Objects.isNull(sourceEvent) || Objects.isNull(sourceEvent.getTableModelDatabaseName()) || AuthorityChecker.getAccessControl() .checkCanSelectFromTable4Pipe( - userName, + entity.getUsername(), new QualifiedObjectName( sourceEvent.getTableModelDatabaseName(), tableName), - new UserEntity(-1, userName, "")); + entity); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java index 4bb1f8d2e177..db017dfaf1da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java @@ -132,7 +132,7 @@ public Optional visitLoadFile( Long.MIN_VALUE, Long.MAX_VALUE, null, - "root", + null, null, true)) { for (final TabletInsertionEvent tabletInsertionEvent : parser.toTabletInsertionEvents()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java index 275bc694397d..9c4c2fe49532 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java @@ -45,6 +45,8 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; +import static org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent.isTabletEmpty; + public class PipeTabletEventTsFileBatch extends PipeTabletEventBatch { private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventTsFileBatch.class); @@ -85,7 +87,7 @@ protected boolean constructBatch(final TabletInsertionEvent event) { final List tablets = insertNodeTabletInsertionEvent.convertToTablets(); for (int i = 0; i < tablets.size(); ++i) { final Tablet tablet = tablets.get(i); - if (tablet.getRowSize() == 0) { + if (isTabletEmpty(tablet)) { continue; } if (isTableModel) { @@ -108,7 +110,7 @@ protected boolean constructBatch(final TabletInsertionEvent event) { final PipeRawTabletInsertionEvent rawTabletInsertionEvent = (PipeRawTabletInsertionEvent) event; final Tablet tablet = rawTabletInsertionEvent.convertToTablet(); - if (tablet.getRowSize() == 0) { + if (isTabletEmpty(tablet)) { return true; } if (rawTabletInsertionEvent.isTableModelEvent()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/legacy/IoTDBLegacyPipeSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/legacy/IoTDBLegacyPipeSink.java index c07d379232ab..eefa5e739d5b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/legacy/IoTDBLegacyPipeSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/legacy/IoTDBLegacyPipeSink.java @@ -88,6 +88,7 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_IOTDB_SYNC_CONNECTOR_VERSION_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_IOTDB_USERNAME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_IOTDB_USER_KEY; +import static org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent.isTabletEmpty; @TreeModel public class IoTDBLegacyPipeSink implements PipeConnector { @@ -325,7 +326,7 @@ private void doTransfer(final PipeInsertNodeTabletInsertionEvent pipeInsertNodeI final List tablets = pipeInsertNodeInsertionEvent.convertToTablets(); for (int i = 0; i < tablets.size(); ++i) { final Tablet tablet = tablets.get(i); - if (Objects.isNull(tablet) || tablet.getRowSize() == 0) { + if (Objects.isNull(tablet) || isTabletEmpty(tablet)) { continue; } if (pipeInsertNodeInsertionEvent.isAligned(i)) { @@ -337,7 +338,7 @@ private void doTransfer(final PipeInsertNodeTabletInsertionEvent pipeInsertNodeI } private void doTransferWrapper(final PipeRawTabletInsertionEvent pipeRawTabletInsertionEvent) - throws PipeException, IoTDBConnectionException, StatementExecutionException { + throws Exception { // We increase the reference count for this event to determine if the event may be released. if (!pipeRawTabletInsertionEvent.increaseReferenceCount(IoTDBLegacyPipeSink.class.getName())) { return; @@ -351,7 +352,7 @@ private void doTransferWrapper(final PipeRawTabletInsertionEvent pipeRawTabletIn } private void doTransfer(final PipeRawTabletInsertionEvent pipeTabletInsertionEvent) - throws PipeException, IoTDBConnectionException, StatementExecutionException { + throws Exception { final Tablet tablet = pipeTabletInsertionEvent.convertToTablet(); if (pipeTabletInsertionEvent.isAligned()) { sessionPool.insertAlignedTablet(tablet); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/writeback/WriteBackSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/writeback/WriteBackSink.java index 225616e88fa9..fddb9f2f1a99 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/writeback/WriteBackSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/writeback/WriteBackSink.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; +import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.auth.AuthorityChecker; @@ -44,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask; import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckContext; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -61,6 +63,7 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.util.concurrent.ListenableFuture; @@ -75,6 +78,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_IOTDB_CLI_HOSTNAME; @@ -105,10 +109,9 @@ public class WriteBackSink implements PipeConnector { // for correctly handling data insertion in IoTDBReceiverAgent#receive method private static final Coordinator COORDINATOR = Coordinator.getInstance(); private static final SessionManager SESSION_MANAGER = SessionManager.getInstance(); - private IClientSession session; + public static final AtomicLong id = new AtomicLong(); + private InternalClientSession session; - // Temporary, used to separate - private IClientSession treeSession; private boolean skipIfNoPrivileges; private boolean useEventUserName; @@ -136,11 +139,12 @@ public void customize( session = new InternalClientSession( String.format( - "%s_%s_%s_%s", + "%s_%s_%s_%s_%s", WriteBackSink.class.getSimpleName(), environment.getPipeName(), environment.getCreationTime(), - environment.getRegionId())); + environment.getRegionId(), + id.getAndIncrement())); String userIdString = parameters.getStringOrDefault( @@ -160,19 +164,6 @@ public void customize( session.setClientVersion(IoTDBConstant.ClientVersion.V_1_0); session.setZoneId(ZoneId.systemDefault()); - // Temporary - treeSession = - new InternalClientSession( - String.format( - "%s_%s_%s_%s_tree", - WriteBackSink.class.getSimpleName(), - environment.getPipeName(), - environment.getCreationTime(), - environment.getRegionId())); - treeSession.setUsername(AuthorityChecker.SUPER_USER); - treeSession.setClientVersion(IoTDBConstant.ClientVersion.V_1_0); - treeSession.setZoneId(ZoneId.systemDefault()); - final String connectorSkipIfValue = parameters .getStringOrDefault( @@ -195,6 +186,10 @@ public void customize( parameters.getBooleanOrDefault( Arrays.asList(CONNECTOR_USE_EVENT_USER_NAME_KEY, SINK_USE_EVENT_USER_NAME_KEY), CONNECTOR_USE_EVENT_USER_NAME_DEFAULT_VALUE); + + if (SESSION_MANAGER.getCurrSession() != null) { + SESSION_MANAGER.registerSession(session); + } } @Override @@ -213,7 +208,7 @@ public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exc if (!(tabletInsertionEvent instanceof PipeInsertNodeTabletInsertionEvent) && !(tabletInsertionEvent instanceof PipeRawTabletInsertionEvent)) { LOGGER.warn( - "WriteBackConnector only support " + "WriteBackSink only support " + "PipeInsertNodeTabletInsertionEvent and PipeRawTabletInsertionEvent. " + "Ignore {}.", tabletInsertionEvent); @@ -368,12 +363,10 @@ private void doTransfer(final PipeStatementInsertionEvent pipeStatementInsertion @Override public void close() throws Exception { - if (session != null) { + if (session != null && SESSION_MANAGER.getCurrSession() == session) { + SESSION_MANAGER.removeCurrSession(); SESSION_MANAGER.closeSession(session, COORDINATOR::cleanupQueryExecution); } - if (treeSession != null) { - SESSION_MANAGER.closeSession(treeSession, COORDINATOR::cleanupQueryExecution); - } } private TSStatus executeStatementForTableModel( @@ -384,7 +377,6 @@ private TSStatus executeStatementForTableModel( if (useEventUserName && userName != null) { session.setUsername(userName); } - SESSION_MANAGER.registerSession(session); try { autoCreateDatabaseIfNecessary(dataBaseName); return Coordinator.getInstance() @@ -436,7 +428,6 @@ private TSStatus executeStatementForTableModel( // If the exception is not caused by database not set, throw it directly throw e; } finally { - SESSION_MANAGER.removeCurrSession(); if (useEventUserName) { session.setUsername(originalUserName); } @@ -485,19 +476,34 @@ private void autoCreateDatabaseIfNecessary(final String database) { } private TSStatus executeStatementForTreeModel(final Statement statement, final String userName) { - treeSession.setDatabaseName(null); - treeSession.setSqlDialect(IClientSession.SqlDialect.TREE); - final String originalUserName = treeSession.getUsername(); + session.setDatabaseName(null); + session.setSqlDialect(IClientSession.SqlDialect.TREE); + final String originalUserName = session.getUsername(); if (useEventUserName && userName != null) { - treeSession.setUsername(userName); + session.setUsername(userName); + } + final TSStatus permissionCheckStatus = + AuthorityChecker.checkAuthority( + statement, + new TreeAccessCheckContext( + session.getUserId(), session.getUsername(), session.getClientAddress())); + if (permissionCheckStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + PipeLogger.log( + LOGGER::warn, + "Session {}: Failed to check authority for statement {}, username = {}, response = {}.", + session.getClientAddress() + ":" + session.getClientPort(), + statement.getType().name(), + session.getUsername(), + permissionCheckStatus); + return RpcUtils.getStatus( + permissionCheckStatus.getCode(), permissionCheckStatus.getMessage()); } - SESSION_MANAGER.registerSession(treeSession); try { return Coordinator.getInstance() .executeForTreeModel( new PipeEnrichedStatement(statement), SESSION_MANAGER.requestQueryId(), - SESSION_MANAGER.getSessionInfo(treeSession), + SESSION_MANAGER.getSessionInfo(session), "", ClusterPartitionFetcher.getInstance(), ClusterSchemaFetcher.getInstance(), @@ -505,9 +511,8 @@ private TSStatus executeStatementForTreeModel(final Statement statement, final S false) .status; } finally { - SESSION_MANAGER.removeCurrSession(); if (useEventUserName) { - treeSession.setUsername(originalUserName); + session.setUsername(originalUserName); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java index 6e675ed0fdd1..284c2d9f396e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java @@ -175,7 +175,13 @@ public Pair, Set } if (event.getEvent() instanceof PipeTsFileInsertionEvent) { - ((PipeTsFileInsertionEvent) event.getEvent()).setTableNames(tableNames); + final PipeTsFileInsertionEvent tsFileInsertionEvent = + (PipeTsFileInsertionEvent) event.getEvent(); + if (tsFileInsertionEvent.isTableModelEvent()) { + tsFileInsertionEvent.setTableNames(tableNames); + } else { + tsFileInsertionEvent.setTreeSchemaMap(event.getSchemaInfo()); + } } return new Pair<>(matchedSources, findUnmatchedSources(matchedSources)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java index f096616e324c..1c44b116a725 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java @@ -73,6 +73,8 @@ public class IoTDBSchemaRegionSource extends IoTDBNonDataRegionSource { private static final PipeStatementToPlanVisitor STATEMENT_TO_PLAN_VISITOR = new PipeStatementToPlanVisitor(); + // Local for exception + private PipePlanTreePrivilegeParseVisitor treePrivilegeParseVisitor; private SchemaRegionId schemaRegionId; private Set listenedTypeSet = new HashSet<>(); @@ -96,6 +98,7 @@ public void customize( schemaRegionId = new SchemaRegionId(regionId); listenedTypeSet = SchemaRegionListeningFilter.parseListeningPlanTypeSet(parameters); + treePrivilegeParseVisitor = new PipePlanTreePrivilegeParseVisitor(skipIfNoPrivileges); PipeSchemaRegionSourceMetrics.getInstance().register(this); PipeDataNodeSinglePipeMetrics.getInstance().register(this); @@ -203,8 +206,9 @@ protected PipeWritePlanEvent getNextEventInCurrentSnapshot() { protected Optional trimRealtimeEventByPrivilege( final PipeWritePlanEvent event) throws AccessDeniedException { final Optional result = - TABLE_PRIVILEGE_PARSE_VISITOR.process( - ((PipeSchemaRegionWritePlanEvent) event).getPlanNode(), userEntity); + treePrivilegeParseVisitor + .process(((PipeSchemaRegionWritePlanEvent) event).getPlanNode(), userEntity) + .flatMap(planNode -> TABLE_PRIVILEGE_PARSE_VISITOR.process(planNode, userEntity)); if (result.isPresent()) { return Optional.of( new PipeSchemaRegionWritePlanEvent(result.get(), event.isGeneratedByPipe())); @@ -225,7 +229,7 @@ protected Optional trimRealtimeEventByPipePattern( .flatMap( planNode -> TABLE_PATTERN_PARSE_VISITOR - .process(((PipeSchemaRegionWritePlanEvent) event).getPlanNode(), tablePattern) + .process(planNode, tablePattern) .map( planNode1 -> new PipeSchemaRegionWritePlanEvent( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePrivilegeParseVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePrivilegeParseVisitor.java new file mode 100644 index 000000000000..a9ff6463e88f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/PipePlanTreePrivilegeParseVisitor.java @@ -0,0 +1,355 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.schemaregion; + +import org.apache.iotdb.commons.audit.IAuditEntity; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.exception.auth.AccessDeniedException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; +import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ActivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.BatchActivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateAlignedTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.InternalBatchActivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.InternalCreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.InternalCreateTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.MeasurementGroup; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.AlterLogicalViewNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.CreateLogicalViewNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckContext; +import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckVisitor; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ActivateTemplateStatement; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.utils.Pair; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class PipePlanTreePrivilegeParseVisitor + extends PlanVisitor, IAuditEntity> { + + private final boolean skip; + + public PipePlanTreePrivilegeParseVisitor(final boolean skip) { + this.skip = skip; + } + + @Override + public Optional visitPlan(final PlanNode node, final IAuditEntity context) { + return Optional.of(node); + } + + @Override + public Optional visitCreateTimeSeries( + final CreateTimeSeriesNode node, final IAuditEntity auditEntity) { + return AuthorityChecker.getAccessControl() + .checkSeriesPrivilege4Pipe( + auditEntity, + Collections.singletonList(node.getPath()), + PrivilegeType.READ_SCHEMA) + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? Optional.of(node) + : Optional.empty(); + } + + @Override + public Optional visitCreateAlignedTimeSeries( + final CreateAlignedTimeSeriesNode node, final IAuditEntity auditEntity) { + final List failedIndexes = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilegeWithIndexes4Pipe( + auditEntity, + node.getMeasurements().stream() + .map(measurement -> node.getDevicePath().concatAsMeasurementPath(measurement)) + .collect(Collectors.toList()), + PrivilegeType.READ_SCHEMA); + if (!skip && !failedIndexes.isEmpty()) { + throw new AccessDeniedException("Not has privilege to transfer plan: " + node); + } + return failedIndexes.size() != node.getMeasurements().size() + ? Optional.of( + new CreateAlignedTimeSeriesNode( + node.getPlanNodeId(), + node.getDevicePath(), + IoTDBTreePattern.applyReversedIndexesOnList(failedIndexes, node.getMeasurements()), + IoTDBTreePattern.applyReversedIndexesOnList(failedIndexes, node.getDataTypes()), + IoTDBTreePattern.applyReversedIndexesOnList(failedIndexes, node.getEncodings()), + IoTDBTreePattern.applyReversedIndexesOnList(failedIndexes, node.getCompressors()), + IoTDBTreePattern.applyReversedIndexesOnList(failedIndexes, node.getAliasList()), + IoTDBTreePattern.applyReversedIndexesOnList(failedIndexes, node.getTagsList()), + IoTDBTreePattern.applyReversedIndexesOnList( + failedIndexes, node.getAttributesList()))) + : Optional.empty(); + } + + @Override + public Optional visitCreateMultiTimeSeries( + final CreateMultiTimeSeriesNode node, final IAuditEntity auditEntity) { + final Map filteredMeasurementGroupMap = + node.getMeasurementGroupMap().entrySet().stream() + .map( + entry -> + new Pair<>( + entry.getKey(), + trimMeasurementGroup(entry.getKey(), entry.getValue(), auditEntity, node))) + .filter(pair -> Objects.nonNull(pair.getRight())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); + return !filteredMeasurementGroupMap.isEmpty() + ? Optional.of( + new CreateMultiTimeSeriesNode(node.getPlanNodeId(), filteredMeasurementGroupMap)) + : Optional.empty(); + } + + private MeasurementGroup trimMeasurementGroup( + final PartialPath device, + final MeasurementGroup group, + final IAuditEntity entity, + final PlanNode node) { + final Set failedIndexes = + new HashSet<>( + AuthorityChecker.getAccessControl() + .checkSeriesPrivilegeWithIndexes4Pipe( + entity, + group.getMeasurements().stream() + .map(device::concatAsMeasurementPath) + .collect(Collectors.toList()), + PrivilegeType.READ_SCHEMA)); + if (!skip && !failedIndexes.isEmpty()) { + throw new AccessDeniedException("Not has privilege to transfer plan: " + node); + } + if (failedIndexes.size() == group.size()) { + return null; + } + final MeasurementGroup targetMeasurementGroup = new MeasurementGroup(); + IntStream.range(0, group.size()) + .filter(index -> !failedIndexes.contains(index)) + .forEach( + index -> { + targetMeasurementGroup.addMeasurement( + group.getMeasurements().get(index), + group.getDataTypes().get(index), + group.getEncodings().get(index), + group.getCompressors().get(index)); + if (Objects.nonNull(group.getTagsList())) { + targetMeasurementGroup.addTags(group.getTagsList().get(index)); + } + if (Objects.nonNull(group.getAttributesList())) { + targetMeasurementGroup.addAttributes(group.getAttributesList().get(index)); + } + if (Objects.nonNull(group.getAliasList())) { + targetMeasurementGroup.addAlias(group.getAliasList().get(index)); + } + if (Objects.nonNull(group.getPropsList())) { + targetMeasurementGroup.addProps(group.getPropsList().get(index)); + } + }); + return targetMeasurementGroup; + } + + @Override + public Optional visitAlterTimeSeries( + final AlterTimeSeriesNode node, final IAuditEntity auditEntity) { + return AuthorityChecker.getAccessControl() + .checkSeriesPrivilege4Pipe( + auditEntity, + Collections.singletonList(node.getPath()), + PrivilegeType.READ_SCHEMA) + .getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? Optional.of(node) + : Optional.empty(); + } + + @Override + public Optional visitInternalCreateTimeSeries( + final InternalCreateTimeSeriesNode node, final IAuditEntity auditEntity) { + final MeasurementGroup group = + trimMeasurementGroup(node.getDevicePath(), node.getMeasurementGroup(), auditEntity, node); + return Objects.nonNull(group) + ? Optional.of( + new InternalCreateTimeSeriesNode( + node.getPlanNodeId(), node.getDevicePath(), group, node.isAligned())) + : Optional.empty(); + } + + @Override + public Optional visitActivateTemplate( + final ActivateTemplateNode node, final IAuditEntity auditEntity) { + final List failedPos = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilegeWithIndexes4Pipe( + auditEntity, + ActivateTemplateStatement.getPaths(node.getActivatePath()), + PrivilegeType.READ_SCHEMA); + if (!failedPos.isEmpty()) { + if (!skip) { + throw new AccessDeniedException("Not has privilege to transfer plan: " + node); + } + return Optional.empty(); + } + return Optional.of(node); + } + + @Override + public Optional visitInternalBatchActivateTemplate( + final InternalBatchActivateTemplateNode node, final IAuditEntity auditEntity) { + final Map> filteredMap = new HashMap<>(); + for (final Map.Entry> pathEntry : + node.getTemplateActivationMap().entrySet()) { + final List failedIndexes = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilegeWithIndexes4Pipe( + auditEntity, + ActivateTemplateStatement.getPaths(pathEntry.getKey()), + PrivilegeType.READ_SCHEMA); + if (failedIndexes.isEmpty()) { + filteredMap.put(pathEntry.getKey(), pathEntry.getValue()); + } else if (!skip) { + throw new AccessDeniedException("Not has privilege to transfer plan: " + node); + } + } + return !filteredMap.isEmpty() + ? Optional.of(new InternalBatchActivateTemplateNode(node.getPlanNodeId(), filteredMap)) + : Optional.empty(); + } + + @Override + public Optional visitInternalCreateMultiTimeSeries( + final InternalCreateMultiTimeSeriesNode node, final IAuditEntity auditEntity) { + final Map> filteredDeviceMap = + node.getDeviceMap().entrySet().stream() + .map( + entry -> + new Pair<>( + entry.getKey(), + new Pair<>( + entry.getValue().getLeft(), + trimMeasurementGroup( + entry.getKey(), entry.getValue().getRight(), auditEntity, node)))) + .filter(pair -> Objects.nonNull(pair.getRight().getRight())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); + return !filteredDeviceMap.isEmpty() + ? Optional.of( + new InternalCreateMultiTimeSeriesNode(node.getPlanNodeId(), filteredDeviceMap)) + : Optional.empty(); + } + + @Override + public Optional visitBatchActivateTemplate( + final BatchActivateTemplateNode node, final IAuditEntity auditEntity) { + final Map> filteredMap = new HashMap<>(); + for (final Map.Entry> pathEntry : + node.getTemplateActivationMap().entrySet()) { + final List failedIndexes = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilegeWithIndexes4Pipe( + auditEntity, + ActivateTemplateStatement.getPaths(pathEntry.getKey()), + PrivilegeType.READ_SCHEMA); + if (failedIndexes.isEmpty()) { + filteredMap.put(pathEntry.getKey(), pathEntry.getValue()); + } else if (!skip) { + throw new AccessDeniedException("Not has privilege to transfer plan: " + node); + } + } + return !filteredMap.isEmpty() + ? Optional.of(new BatchActivateTemplateNode(node.getPlanNodeId(), filteredMap)) + : Optional.empty(); + } + + @Override + public Optional visitCreateLogicalView( + final CreateLogicalViewNode node, final IAuditEntity auditEntity) { + final Map filteredMap = + new HashMap<>(node.getViewPathToSourceExpressionMap()); + final List viewPathList = node.getViewPathList(); + final List failedIndexes = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilegeWithIndexes4Pipe( + auditEntity, viewPathList, PrivilegeType.READ_SCHEMA); + if (!skip && !failedIndexes.isEmpty()) { + throw new AccessDeniedException("Not has privilege to transfer plan: " + node); + } + failedIndexes.forEach(index -> filteredMap.remove(viewPathList.get(index))); + return !filteredMap.isEmpty() + ? Optional.of(new CreateLogicalViewNode(node.getPlanNodeId(), filteredMap)) + : Optional.empty(); + } + + @Override + public Optional visitAlterLogicalView( + final AlterLogicalViewNode node, final IAuditEntity auditEntity) { + final Map filteredMap = + new HashMap<>(node.getViewPathToSourceMap()); + final List viewPathList = new ArrayList<>(node.getViewPathToSourceMap().keySet()); + final List failedIndexes = + AuthorityChecker.getAccessControl() + .checkSeriesPrivilegeWithIndexes4Pipe( + auditEntity, viewPathList, PrivilegeType.READ_SCHEMA); + if (!skip && !failedIndexes.isEmpty()) { + throw new AccessDeniedException("Not has privilege to transfer plan: " + node); + } + failedIndexes.forEach(index -> filteredMap.remove(viewPathList.get(index))); + return !filteredMap.isEmpty() + ? Optional.of(new AlterLogicalViewNode(node.getPlanNodeId(), filteredMap)) + : Optional.empty(); + } + + @Override + public Optional visitDeleteData( + final DeleteDataNode node, final IAuditEntity auditEntity) { + final List intersectedPaths = + TreeAccessCheckVisitor.getIntersectedPaths4Pipe( + node.getPathList(), + new TreeAccessCheckContext( + auditEntity.getUserId(), auditEntity.getUsername(), auditEntity.getCliHostname())); + if (!skip && !intersectedPaths.equals(node.getPathList())) { + throw new AccessDeniedException("Not has privilege to transfer plan: " + node); + } + return !intersectedPaths.isEmpty() + ? Optional.of( + new DeleteDataNode( + node.getPlanNodeId(), + intersectedPaths, + node.getDeleteStartTime(), + node.getDeleteEndTime())) + : Optional.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java index 05932e78b95a..54a7f6fc8c78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; @@ -36,7 +37,6 @@ import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeVisitorWithLimitOffsetWrapper; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java index 543991a6ff72..5b18b0474b60 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ISchemaTree.java @@ -21,8 +21,8 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Accountable; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitor.java index f1649223117b..d05d1a1ff2a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitor.java @@ -21,10 +21,10 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.tree.AbstractTreeVisitor; import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaMeasurementNode; import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.write.schema.IMeasurementSchema; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorWithLimitOffsetWrapper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorWithLimitOffsetWrapper.java index 575a0062399a..208b63a62ae4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorWithLimitOffsetWrapper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorWithLimitOffsetWrapper.java @@ -19,8 +19,8 @@ package org.apache.iotdb.db.queryengine.common.schematree.visitor; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.ArrayList; import java.util.List; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java index f4aca5e157e1..26a710a85cdc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.ConsensusFactory; @@ -73,7 +74,6 @@ import org.apache.iotdb.db.schemaengine.SchemaEngine; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.trigger.executor.TriggerFireResult; import org.apache.iotdb.db.trigger.executor.TriggerFireVisitor; import org.apache.iotdb.rpc.RpcUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java index 9a8dbf9f21b3..b2ed81a71ae9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; @@ -31,7 +32,6 @@ import org.apache.iotdb.db.queryengine.execution.operator.source.SourceOperator; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.read.common.block.TsBlock; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java index f25005815ef7..2ef0ab9e18a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java @@ -25,10 +25,10 @@ import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.INodeSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.List; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java index 5c13c958c022..40799c548eeb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java @@ -27,13 +27,13 @@ import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.ViewType; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.SchemaRegionReadPlanFactory; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaUtils; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.utils.Pair; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java index 9235bc350889..37603ee2dbca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.NodeRef; @@ -58,7 +59,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 30e8426c707e..af8fa9f2b295 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -36,6 +36,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.utils.TimePartitionUtils; @@ -146,7 +147,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -3295,7 +3295,7 @@ public Analysis visitShowCluster( } @Override - public Analysis visitCountStorageGroup( + public Analysis visitCountDatabase( CountDatabaseStatement countDatabaseStatement, MPPQueryContext context) { Analysis analysis = new Analysis(); analysis.setRealStatement(countDatabaseStatement); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java index dc7df32007f9..3fc8b7e5339a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.analyze; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.NodeRef; @@ -30,7 +31,6 @@ import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; import org.apache.iotdb.db.queryengine.plan.statement.component.ResultColumn; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java index 4a4fc073410b..21b8d9d8dc10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; @@ -36,7 +37,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.component.ResultColumn; import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java index 358614c9a58e..6806b224778b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -44,7 +45,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.AlterSchemaTemplateStatement; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java index 1a2b10d34489..d6f5fd74fa98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.session.SessionManager; @@ -39,7 +40,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.internal.DeviceSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java index 49e2995b3a80..1be03d479b6b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.schema.table.Audit; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -37,7 +38,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java index 87da0e9f0c9b..f56c3e2c92f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java @@ -21,9 +21,9 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/TemplateSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/TemplateSchemaFetcher.java index 2a61b546d5e2..bca0b54ff1e6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/TemplateSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/TemplateSchemaFetcher.java @@ -20,12 +20,12 @@ package org.apache.iotdb.db.queryengine.plan.analyze.schema; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; import org.apache.tsfile.common.conf.TSFileDescriptor; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 0475a27c26da..90923174c2c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -1041,52 +1041,51 @@ private int parseIntFromLiteral(final Object value, final String name) { @Override protected IConfigTask visitCreatePipe(final CreatePipe node, final MPPQueryContext context) { context.setQueryType(QueryType.WRITE); - final String userName = context.getSession().getUserName(); accessControl.checkUserGlobalSysPrivilege(context); - final Map extractorAttributes = node.getExtractorAttributes(); + final Map sourceAttributes = node.getSourceAttributes(); final String pipeName = node.getPipeName(); - for (final String ExtractorAttribute : extractorAttributes.keySet()) { - if (ExtractorAttribute.startsWith(SystemConstant.SYSTEM_PREFIX_KEY)) { + for (final String sourceAttribute : sourceAttributes.keySet()) { + if (sourceAttribute.startsWith(SystemConstant.SYSTEM_PREFIX_KEY)) { throw new SemanticException( String.format( "Failed to create pipe %s, setting %s is not allowed.", - node.getPipeName(), ExtractorAttribute)); + node.getPipeName(), sourceAttribute)); } - if (ExtractorAttribute.startsWith(SystemConstant.AUDIT_PREFIX_KEY)) { + if (sourceAttribute.startsWith(SystemConstant.AUDIT_PREFIX_KEY)) { throw new SemanticException( String.format( "Failed to create pipe %s, setting %s is not allowed.", - node.getPipeName(), ExtractorAttribute)); + node.getPipeName(), sourceAttribute)); } } // Inject table model into the extractor attributes - extractorAttributes.put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); + sourceAttributes.put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); checkAndEnrichSourceUser( pipeName, - extractorAttributes, + sourceAttributes, new UserEntity(context.getUserId(), context.getUsername(), context.getCliHostname()), false); checkAndEnrichSinkUser( pipeName, - node.getConnectorAttributes(), + node.getSinkAttributes(), new UserEntity(context.getUserId(), context.getUsername(), context.getCliHostname()), false); - mayChangeSourcePattern(extractorAttributes); + mayChangeSourcePattern(sourceAttributes); return new CreatePipeTask(node); } public static void checkAndEnrichSourceUser( final String pipeName, - final Map replacedExtractorAttributes, + final Map replacedSourceAttributes, final UserEntity userEntity, final boolean isAlter) { - final PipeParameters extractorParameters = new PipeParameters(replacedExtractorAttributes); + final PipeParameters sourceParameters = new PipeParameters(replacedSourceAttributes); final String pluginName = - extractorParameters + sourceParameters .getStringOrDefault( Arrays.asList(PipeSourceConstant.EXTRACTOR_KEY, PipeSourceConstant.SOURCE_KEY), BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName()) @@ -1097,18 +1096,18 @@ public static void checkAndEnrichSourceUser( return; } - if (!extractorParameters.hasAnyAttributes( + if (!sourceParameters.hasAnyAttributes( PipeSourceConstant.EXTRACTOR_IOTDB_USER_KEY, PipeSourceConstant.SOURCE_IOTDB_USER_KEY, PipeSourceConstant.EXTRACTOR_IOTDB_USERNAME_KEY, PipeSourceConstant.SOURCE_IOTDB_USERNAME_KEY)) { - replacedExtractorAttributes.put( + replacedSourceAttributes.put( PipeSourceConstant.SOURCE_IOTDB_USER_ID, String.valueOf(userEntity.getUserId())); - replacedExtractorAttributes.put( + replacedSourceAttributes.put( PipeSourceConstant.SOURCE_IOTDB_USERNAME_KEY, userEntity.getUsername()); - replacedExtractorAttributes.put( + replacedSourceAttributes.put( PipeSourceConstant.SOURCE_IOTDB_CLI_HOSTNAME, userEntity.getCliHostname()); - } else if (!extractorParameters.hasAnyAttributes( + } else if (!sourceParameters.hasAnyAttributes( PipeSourceConstant.EXTRACTOR_IOTDB_PASSWORD_KEY, PipeSourceConstant.SOURCE_IOTDB_PASSWORD_KEY)) { throw new SemanticException( @@ -1118,11 +1117,11 @@ public static void checkAndEnrichSourceUser( } } - private static void mayChangeSourcePattern(final Map extractorAttributes) { - final PipeParameters extractorParameters = new PipeParameters(extractorAttributes); + private static void mayChangeSourcePattern(final Map sourceAttributes) { + final PipeParameters sourceParameters = new PipeParameters(sourceAttributes); final String pluginName = - extractorParameters + sourceParameters .getStringOrDefault( Arrays.asList(PipeSourceConstant.EXTRACTOR_KEY, PipeSourceConstant.SOURCE_KEY), BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName()) @@ -1134,14 +1133,14 @@ private static void mayChangeSourcePattern(final Map extractorAt } // Use lower case because database + table name are all in lower cases - extractorParameters.computeAttributeIfExists( + sourceParameters.computeAttributeIfExists( (k, v) -> v.toLowerCase(Locale.ENGLISH), PipeSourceConstant.EXTRACTOR_DATABASE_KEY, PipeSourceConstant.SOURCE_DATABASE_KEY, PipeSourceConstant.EXTRACTOR_DATABASE_NAME_KEY, PipeSourceConstant.SOURCE_DATABASE_NAME_KEY); - extractorParameters.computeAttributeIfExists( + sourceParameters.computeAttributeIfExists( (k, v) -> v.toLowerCase(Locale.ENGLISH), PipeSourceConstant.EXTRACTOR_TABLE_KEY, PipeSourceConstant.SOURCE_TABLE_KEY, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index 465326ff6394..5c5c3f6b171e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -250,20 +250,18 @@ public IConfigTask visitAlterDatabase( } @Override - public IConfigTask visitDeleteStorageGroup( + public IConfigTask visitDeleteDatabase( DeleteDatabaseStatement statement, MPPQueryContext context) { return new DeleteStorageGroupTask(statement); } @Override - public IConfigTask visitShowStorageGroup( - ShowDatabaseStatement statement, MPPQueryContext context) { + public IConfigTask visitShowDatabase(ShowDatabaseStatement statement, MPPQueryContext context) { return new ShowDatabaseTask(statement); } @Override - public IConfigTask visitCountStorageGroup( - CountDatabaseStatement statement, MPPQueryContext context) { + public IConfigTask visitCountDatabase(CountDatabaseStatement statement, MPPQueryContext context) { return new CountDatabaseTask(statement); } @@ -565,7 +563,7 @@ public IConfigTask visitShowAINodes( @Override public IConfigTask visitShowPipes( ShowPipesStatement showPipesStatement, MPPQueryContext context) { - return new ShowPipeTask(showPipesStatement); + return new ShowPipeTask(showPipesStatement, context.getUsername()); } @Override @@ -576,33 +574,33 @@ public IConfigTask visitDropPipe(DropPipeStatement dropPipeStatement, MPPQueryCo @Override public IConfigTask visitCreatePipe( final CreatePipeStatement createPipeStatement, final MPPQueryContext context) { - for (final String ExtractorAttribute : createPipeStatement.getExtractorAttributes().keySet()) { - if (ExtractorAttribute.startsWith(SystemConstant.SYSTEM_PREFIX_KEY)) { + for (final String sourceAttribute : createPipeStatement.getSourceAttributes().keySet()) { + if (sourceAttribute.startsWith(SystemConstant.SYSTEM_PREFIX_KEY)) { throw new SemanticException( String.format( "Failed to create pipe %s, setting %s is not allowed.", - createPipeStatement.getPipeName(), ExtractorAttribute)); + createPipeStatement.getPipeName(), sourceAttribute)); } - if (ExtractorAttribute.startsWith(SystemConstant.AUDIT_PREFIX_KEY)) { + if (sourceAttribute.startsWith(SystemConstant.AUDIT_PREFIX_KEY)) { throw new SemanticException( String.format( "Failed to create pipe %s, setting %s is not allowed.", - createPipeStatement.getPipeName(), ExtractorAttribute)); + createPipeStatement.getPipeName(), sourceAttribute)); } } - // Inject tree model into the extractor attributes + // Inject tree model into the source attributes createPipeStatement - .getExtractorAttributes() + .getSourceAttributes() .put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); checkAndEnrichSourceUser( createPipeStatement.getPipeName(), - createPipeStatement.getExtractorAttributes(), + createPipeStatement.getSourceAttributes(), new UserEntity(context.getUserId(), context.getUsername(), context.getCliHostname()), false); checkAndEnrichSinkUser( createPipeStatement.getPipeName(), - createPipeStatement.getConnectorAttributes(), + createPipeStatement.getSinkAttributes(), context.getSession().getUserEntity(), false); @@ -613,8 +611,7 @@ public IConfigTask visitCreatePipe( public IConfigTask visitAlterPipe( final AlterPipeStatement alterPipeStatement, final MPPQueryContext context) { - for (final String extractorAttributeKey : - alterPipeStatement.getExtractorAttributes().keySet()) { + for (final String extractorAttributeKey : alterPipeStatement.getSourceAttributes().keySet()) { if (extractorAttributeKey.startsWith(SystemConstant.SYSTEM_PREFIX_KEY)) { throw new SemanticException( String.format( @@ -633,11 +630,11 @@ public IConfigTask visitAlterPipe( alterPipeStatement.setUserName(userName); final String pipeName = alterPipeStatement.getPipeName(); - final Map extractorAttributes = alterPipeStatement.getExtractorAttributes(); + final Map extractorAttributes = alterPipeStatement.getSourceAttributes(); // If the source is replaced, sql-dialect uses the current Alter Pipe sql-dialect. If it is // modified, the original sql-dialect is used. - if (alterPipeStatement.isReplaceAllExtractorAttributes()) { + if (alterPipeStatement.isReplaceAllSourceAttributes()) { extractorAttributes.put( SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); checkAndEnrichSourceUser( @@ -647,10 +644,10 @@ public IConfigTask visitAlterPipe( true); } - if (alterPipeStatement.isReplaceAllConnectorAttributes()) { + if (alterPipeStatement.isReplaceAllSinkAttributes()) { checkAndEnrichSinkUser( pipeName, - alterPipeStatement.getConnectorAttributes(), + alterPipeStatement.getSinkAttributes(), context.getSession().getUserEntity(), true); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 8770ed2873ce..da00a21e9991 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -66,6 +66,7 @@ import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; @@ -300,7 +301,6 @@ import org.apache.iotdb.db.schemaengine.rescon.DataNodeSchemaQuotaManager; import org.apache.iotdb.db.schemaengine.table.InformationSchemaUtils; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; import org.apache.iotdb.db.schemaengine.template.alter.TemplateAlterOperationUtil; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; @@ -2094,9 +2094,9 @@ public SettableFuture createPipe( PipeDataNodeAgent.plugin() .validate( createPipeStatement.getPipeName(), - createPipeStatement.getExtractorAttributes(), + createPipeStatement.getSourceAttributes(), createPipeStatement.getProcessorAttributes(), - createPipeStatement.getConnectorAttributes()); + createPipeStatement.getSinkAttributes()); } catch (final Exception e) { future.setException( new IoTDBException(e.getMessage(), TSStatusCode.PIPE_ERROR.getStatusCode())); @@ -2106,7 +2106,7 @@ public SettableFuture createPipe( // Syntactic sugar: if full-sync mode is detected (i.e. not snapshot mode, or both realtime // and history are true), the pipe is split into history-only and realtime–only modes. final PipeParameters sourcePipeParameters = - new PipeParameters(createPipeStatement.getExtractorAttributes()); + new PipeParameters(createPipeStatement.getSourceAttributes()); if (PipeConfig.getInstance().getPipeAutoSplitFullEnabled() && PipeDataNodeAgent.task().isFullSync(sourcePipeParameters)) { try (final ConfigNodeClient configNodeClient = @@ -2130,7 +2130,7 @@ public SettableFuture createPipe( Boolean.toString(false)))) .getAttribute()) .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) - .setConnectorAttributes(createPipeStatement.getConnectorAttributes()); + .setConnectorAttributes(createPipeStatement.getSinkAttributes()); final TSStatus realtimeTsStatus = configNodeClient.createPipe(realtimeReq); // If creation fails, immediately return with exception @@ -2164,7 +2164,7 @@ public SettableFuture createPipe( PipeSourceConstant.EXTRACTOR_EXCLUSION_DEFAULT_VALUE))) .getAttribute()) .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) - .setConnectorAttributes(createPipeStatement.getConnectorAttributes()); + .setConnectorAttributes(createPipeStatement.getSinkAttributes()); final TSStatus historyTsStatus = configNodeClient.createPipe(historyReq); // If creation fails, immediately return with exception @@ -2188,9 +2188,9 @@ public SettableFuture createPipe( new TCreatePipeReq() .setPipeName(createPipeStatement.getPipeName()) .setIfNotExistsCondition(createPipeStatement.hasIfNotExistsCondition()) - .setExtractorAttributes(createPipeStatement.getExtractorAttributes()) + .setExtractorAttributes(createPipeStatement.getSourceAttributes()) .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) - .setConnectorAttributes(createPipeStatement.getConnectorAttributes()); + .setConnectorAttributes(createPipeStatement.getSinkAttributes()); final TSStatus tsStatus = configNodeClient.createPipe(req); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus)); @@ -2269,29 +2269,25 @@ public SettableFuture alterPipe(final AlterPipeStatement alter final Map processorAttributes; final Map connectorAttributes; try { - if (!alterPipeStatement.getExtractorAttributes().isEmpty()) { + if (!alterPipeStatement.getSourceAttributes().isEmpty()) { // We don't allow changing the extractor plugin type - if (alterPipeStatement - .getExtractorAttributes() - .containsKey(PipeSourceConstant.EXTRACTOR_KEY) - || alterPipeStatement - .getExtractorAttributes() - .containsKey(PipeSourceConstant.SOURCE_KEY) - || alterPipeStatement.isReplaceAllExtractorAttributes()) { + if (alterPipeStatement.getSourceAttributes().containsKey(PipeSourceConstant.EXTRACTOR_KEY) + || alterPipeStatement.getSourceAttributes().containsKey(PipeSourceConstant.SOURCE_KEY) + || alterPipeStatement.isReplaceAllSourceAttributes()) { checkIfSourcePluginChanged( pipeMetaFromCoordinator.getStaticMeta().getSourceParameters(), - new PipeParameters(alterPipeStatement.getExtractorAttributes())); + new PipeParameters(alterPipeStatement.getSourceAttributes())); } - if (alterPipeStatement.isReplaceAllExtractorAttributes()) { - extractorAttributes = alterPipeStatement.getExtractorAttributes(); + if (alterPipeStatement.isReplaceAllSourceAttributes()) { + extractorAttributes = alterPipeStatement.getSourceAttributes(); } else { final boolean onlyContainsUser = - onlyContainsUser(alterPipeStatement.getExtractorAttributes()); + onlyContainsUser(alterPipeStatement.getSourceAttributes()); pipeMetaFromCoordinator .getStaticMeta() .getSourceParameters() .addOrReplaceEquivalentAttributes( - new PipeParameters(alterPipeStatement.getExtractorAttributes())); + new PipeParameters(alterPipeStatement.getSourceAttributes())); extractorAttributes = pipeMetaFromCoordinator.getStaticMeta().getSourceParameters().getAttribute(); if (onlyContainsUser) { @@ -2320,17 +2316,16 @@ public SettableFuture alterPipe(final AlterPipeStatement alter pipeMetaFromCoordinator.getStaticMeta().getProcessorParameters().getAttribute(); } - if (!alterPipeStatement.getConnectorAttributes().isEmpty()) { - if (alterPipeStatement.isReplaceAllConnectorAttributes()) { - connectorAttributes = alterPipeStatement.getConnectorAttributes(); + if (!alterPipeStatement.getSinkAttributes().isEmpty()) { + if (alterPipeStatement.isReplaceAllSinkAttributes()) { + connectorAttributes = alterPipeStatement.getSinkAttributes(); } else { - final boolean onlyContainsUser = - onlyContainsUser(alterPipeStatement.getConnectorAttributes()); + final boolean onlyContainsUser = onlyContainsUser(alterPipeStatement.getSinkAttributes()); pipeMetaFromCoordinator .getStaticMeta() .getSinkParameters() .addOrReplaceEquivalentAttributes( - new PipeParameters(alterPipeStatement.getConnectorAttributes())); + new PipeParameters(alterPipeStatement.getSinkAttributes())); connectorAttributes = pipeMetaFromCoordinator.getStaticMeta().getSinkParameters().getAttribute(); if (onlyContainsUser) { @@ -2356,11 +2351,11 @@ public SettableFuture alterPipe(final AlterPipeStatement alter new TAlterPipeReq( pipeName, alterPipeStatement.getProcessorAttributes(), - alterPipeStatement.getConnectorAttributes(), + alterPipeStatement.getSinkAttributes(), alterPipeStatement.isReplaceAllProcessorAttributes(), - alterPipeStatement.isReplaceAllConnectorAttributes()); - req.setExtractorAttributes(alterPipeStatement.getExtractorAttributes()); - req.setIsReplaceAllExtractorAttributes(alterPipeStatement.isReplaceAllExtractorAttributes()); + alterPipeStatement.isReplaceAllSinkAttributes()); + req.setExtractorAttributes(alterPipeStatement.getSourceAttributes()); + req.setIsReplaceAllExtractorAttributes(alterPipeStatement.isReplaceAllSourceAttributes()); req.setIfExistsCondition(alterPipeStatement.hasIfExistsCondition()); req.setIsTableModel(alterPipeStatement.isTableModel()); final TSStatus tsStatus = configNodeClient.alterPipe(req); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/template/ShowNodesInSchemaTemplateTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/template/ShowNodesInSchemaTemplateTask.java index 9d833e890e5a..e601c3533ef9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/template/ShowNodesInSchemaTemplateTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/template/ShowNodesInSchemaTemplateTask.java @@ -21,13 +21,13 @@ import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ShowNodesInSchemaTemplateStatement; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.util.concurrent.ListenableFuture; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/template/ShowSchemaTemplateTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/template/ShowSchemaTemplateTask.java index 258efb6471c8..870b4c0c076b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/template/ShowSchemaTemplateTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/template/ShowSchemaTemplateTask.java @@ -21,13 +21,13 @@ import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ShowSchemaTemplateStatement; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.util.concurrent.ListenableFuture; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java index c80961ea620d..aecadff7f81a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java @@ -39,7 +39,7 @@ public class AlterPipeTask implements IConfigTask { public AlterPipeTask(final AlterPipeStatement alterPipeStatement) { // support now() function - applyNowFunctionToExtractorAttributes(alterPipeStatement.getExtractorAttributes()); + applyNowFunctionToExtractorAttributes(alterPipeStatement.getSourceAttributes()); this.alterPipeStatement = alterPipeStatement; } @@ -51,12 +51,12 @@ public AlterPipeTask(final AlterPipe node, final String userName) { // support now() function applyNowFunctionToExtractorAttributes(node.getExtractorAttributes()); - alterPipeStatement.setExtractorAttributes(node.getExtractorAttributes()); + alterPipeStatement.setSourceAttributes(node.getExtractorAttributes()); alterPipeStatement.setProcessorAttributes(node.getProcessorAttributes()); - alterPipeStatement.setConnectorAttributes(node.getConnectorAttributes()); - alterPipeStatement.setReplaceAllExtractorAttributes(node.isReplaceAllExtractorAttributes()); + alterPipeStatement.setSinkAttributes(node.getConnectorAttributes()); + alterPipeStatement.setReplaceAllSourceAttributes(node.isReplaceAllExtractorAttributes()); alterPipeStatement.setReplaceAllProcessorAttributes(node.isReplaceAllProcessorAttributes()); - alterPipeStatement.setReplaceAllConnectorAttributes(node.isReplaceAllConnectorAttributes()); + alterPipeStatement.setReplaceAllSinkAttributes(node.isReplaceAllConnectorAttributes()); alterPipeStatement.setUserName(userName); alterPipeStatement.setTableModel(true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/CreatePipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/CreatePipeTask.java index 6e1344ad69d9..a5fbd36f88d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/CreatePipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/CreatePipeTask.java @@ -39,7 +39,7 @@ public class CreatePipeTask implements IConfigTask { public CreatePipeTask(CreatePipeStatement createPipeStatement) { // support now() function - applyNowFunctionToExtractorAttributes(createPipeStatement.getExtractorAttributes()); + applyNowFunctionToExtractorAttributes(createPipeStatement.getSourceAttributes()); this.createPipeStatement = createPipeStatement; } @@ -49,11 +49,11 @@ public CreatePipeTask(CreatePipe createPipe) { createPipeStatement.setIfNotExists(createPipe.hasIfNotExistsCondition()); // support now() function - applyNowFunctionToExtractorAttributes(createPipe.getExtractorAttributes()); + applyNowFunctionToExtractorAttributes(createPipe.getSourceAttributes()); - createPipeStatement.setExtractorAttributes(createPipe.getExtractorAttributes()); + createPipeStatement.setSourceAttributes(createPipe.getSourceAttributes()); createPipeStatement.setProcessorAttributes(createPipe.getProcessorAttributes()); - createPipeStatement.setConnectorAttributes(createPipe.getConnectorAttributes()); + createPipeStatement.setSinkAttributes(createPipe.getSinkAttributes()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java index 445b5320ba14..09b925d3978c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java @@ -47,10 +47,11 @@ public class ShowPipeTask implements IConfigTask { private final ShowPipesStatement showPipesStatement; - private String userName; + private final String userName; - public ShowPipeTask(final ShowPipesStatement showPipesStatement) { + public ShowPipeTask(final ShowPipesStatement showPipesStatement, final String userName) { this.showPipesStatement = showPipesStatement; + this.userName = userName; } public ShowPipeTask(final ShowPipes node, final String userName) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 1cb0b9669841..a25a263490c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -42,7 +42,6 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser; -import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ConnectorAttributeClauseContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ConstantContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountDatabasesContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountDevicesContext; @@ -51,7 +50,6 @@ import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CreateFunctionContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.DropFunctionContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ExpressionContext; -import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ExtractorAttributeClauseContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.GroupByAttributeClauseContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.IdentifierContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ProcessorAttributeClauseContext; @@ -3997,8 +3995,9 @@ public Statement visitCreatePipe(final IoTDBSqlParser.CreatePipeContext ctx) { final CreatePipeStatement createPipeStatement = new CreatePipeStatement(StatementType.CREATE_PIPE); + final String pipeName = parseIdentifier(ctx.pipeName.getText()); if (ctx.pipeName != null) { - createPipeStatement.setPipeName(parseIdentifier(ctx.pipeName.getText())); + createPipeStatement.setPipeName(pipeName); } else { throw new SemanticException( "Not support for this sql in CREATE PIPE, please enter pipe name."); @@ -4007,12 +4006,11 @@ public Statement visitCreatePipe(final IoTDBSqlParser.CreatePipeContext ctx) { createPipeStatement.setIfNotExists( ctx.IF() != null && ctx.NOT() != null && ctx.EXISTS() != null); - if (ctx.extractorAttributesClause() != null) { - createPipeStatement.setExtractorAttributes( - parseExtractorAttributesClause( - ctx.extractorAttributesClause().extractorAttributeClause())); + if (ctx.sourceAttributesClause() != null) { + createPipeStatement.setSourceAttributes( + parseSourceAttributesClause(ctx.sourceAttributesClause().sourceAttributeClause())); } else { - createPipeStatement.setExtractorAttributes(new HashMap<>()); + createPipeStatement.setSourceAttributes(new HashMap<>()); } if (ctx.processorAttributesClause() != null) { createPipeStatement.setProcessorAttributes( @@ -4021,15 +4019,15 @@ public Statement visitCreatePipe(final IoTDBSqlParser.CreatePipeContext ctx) { } else { createPipeStatement.setProcessorAttributes(new HashMap<>()); } - if (ctx.connectorAttributesClause() != null) { - createPipeStatement.setConnectorAttributes( - parseConnectorAttributesClause( - ctx.connectorAttributesClause().connectorAttributeClause())); + if (ctx.sinkAttributesClause() != null) { + createPipeStatement.setSinkAttributes( + parseSinkAttributesClause(ctx.sinkAttributesClause().sinkAttributeClause())); } else { - createPipeStatement.setConnectorAttributes( - parseConnectorAttributesClause( - ctx.connectorAttributesWithoutWithSinkClause().connectorAttributeClause())); + createPipeStatement.setSinkAttributes( + parseSinkAttributesClause( + ctx.sinkAttributesWithoutWithSinkClause().sinkAttributeClause())); } + return createPipeStatement; } @@ -4046,15 +4044,14 @@ public Statement visitAlterPipe(IoTDBSqlParser.AlterPipeContext ctx) { alterPipeStatement.setIfExists(ctx.IF() != null && ctx.EXISTS() != null); - if (ctx.alterExtractorAttributesClause() != null) { - alterPipeStatement.setExtractorAttributes( - parseExtractorAttributesClause( - ctx.alterExtractorAttributesClause().extractorAttributeClause())); - alterPipeStatement.setReplaceAllExtractorAttributes( - Objects.nonNull(ctx.alterExtractorAttributesClause().REPLACE())); + if (ctx.alterSourceAttributesClause() != null) { + alterPipeStatement.setSourceAttributes( + parseSourceAttributesClause(ctx.alterSourceAttributesClause().sourceAttributeClause())); + alterPipeStatement.setReplaceAllSourceAttributes( + Objects.nonNull(ctx.alterSourceAttributesClause().REPLACE())); } else { - alterPipeStatement.setExtractorAttributes(new HashMap<>()); - alterPipeStatement.setReplaceAllExtractorAttributes(false); + alterPipeStatement.setSourceAttributes(new HashMap<>()); + alterPipeStatement.setReplaceAllSourceAttributes(false); } if (ctx.alterProcessorAttributesClause() != null) { @@ -4068,27 +4065,26 @@ public Statement visitAlterPipe(IoTDBSqlParser.AlterPipeContext ctx) { alterPipeStatement.setReplaceAllProcessorAttributes(false); } - if (ctx.alterConnectorAttributesClause() != null) { - alterPipeStatement.setConnectorAttributes( - parseConnectorAttributesClause( - ctx.alterConnectorAttributesClause().connectorAttributeClause())); - alterPipeStatement.setReplaceAllConnectorAttributes( - Objects.nonNull(ctx.alterConnectorAttributesClause().REPLACE())); + if (ctx.alterSinkAttributesClause() != null) { + alterPipeStatement.setSinkAttributes( + parseSinkAttributesClause(ctx.alterSinkAttributesClause().sinkAttributeClause())); + alterPipeStatement.setReplaceAllSinkAttributes( + Objects.nonNull(ctx.alterSinkAttributesClause().REPLACE())); } else { - alterPipeStatement.setConnectorAttributes(new HashMap<>()); - alterPipeStatement.setReplaceAllConnectorAttributes(false); + alterPipeStatement.setSinkAttributes(new HashMap<>()); + alterPipeStatement.setReplaceAllSinkAttributes(false); } return alterPipeStatement; } - private Map parseExtractorAttributesClause( - List contexts) { + private Map parseSourceAttributesClause( + List contexts) { final Map collectorMap = new HashMap<>(); - for (IoTDBSqlParser.ExtractorAttributeClauseContext context : contexts) { + for (IoTDBSqlParser.SourceAttributeClauseContext context : contexts) { collectorMap.put( - parseStringLiteral(context.extractorKey.getText()), - parseStringLiteral(context.extractorValue.getText())); + parseStringLiteral(context.sourceKey.getText()), + parseStringLiteral(context.sourceValue.getText())); } return collectorMap; } @@ -4104,15 +4100,15 @@ private Map parseProcessorAttributesClause( return processorMap; } - private Map parseConnectorAttributesClause( - List contexts) { - final Map connectorMap = new HashMap<>(); - for (IoTDBSqlParser.ConnectorAttributeClauseContext context : contexts) { - connectorMap.put( - parseStringLiteral(context.connectorKey.getText()), - parseStringLiteral(context.connectorValue.getText())); + private Map parseSinkAttributesClause( + List contexts) { + final Map SinkMap = new HashMap<>(); + for (IoTDBSqlParser.SinkAttributeClauseContext context : contexts) { + SinkMap.put( + parseStringLiteral(context.sinkKey.getText()), + parseStringLiteral(context.sinkValue.getText())); } - return connectorMap; + return SinkMap; } @Override @@ -4163,7 +4159,7 @@ public Statement visitShowPipes(IoTDBSqlParser.ShowPipesContext ctx) { if (ctx.pipeName != null) { showPipesStatement.setPipeName(parseIdentifier(ctx.pipeName.getText())); } - showPipesStatement.setWhereClause(ctx.CONNECTOR() != null); + showPipesStatement.setWhereClause(ctx.WHERE() != null); return showPipesStatement; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 7fcd8000c2d5..5414b5ac0200 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.TimeseriesContext; @@ -101,7 +102,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaUtils; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.db.utils.columngenerator.parameter.SlidingTimeColumnGeneratorParameter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index fa39b3ba7dce..2192435b4c29 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; @@ -86,7 +87,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/LevelTimeSeriesCountNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/LevelTimeSeriesCountNode.java index c267736b466d..175a57ce85a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/LevelTimeSeriesCountNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/LevelTimeSeriesCountNode.java @@ -25,10 +25,10 @@ import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/SeriesSchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/SeriesSchemaFetchScanNode.java index c58026b9e13f..df922904a99c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/SeriesSchemaFetchScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/SeriesSchemaFetchScanNode.java @@ -22,12 +22,12 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeUtil; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TimeSeriesCountNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TimeSeriesCountNode.java index 4a935e4d0cca..0e899fcd919a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TimeSeriesCountNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TimeSeriesCountNode.java @@ -26,10 +26,10 @@ import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TimeSeriesSchemaScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TimeSeriesSchemaScanNode.java index 0348234f34b5..1997a3caefe8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TimeSeriesSchemaScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TimeSeriesSchemaScanNode.java @@ -25,10 +25,10 @@ import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/cache/DeviceSchemaRequestCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/cache/DeviceSchemaRequestCache.java index 121d3b515b70..b2711779fccc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/cache/DeviceSchemaRequestCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/cache/DeviceSchemaRequestCache.java @@ -44,11 +44,11 @@ public static DeviceSchemaRequestCache getInstance() { return INSTANCE; } - public FetchMissingDeviceSchema getOrCreatePendingRequest(FetchDevice statement) { + public FetchMissingDeviceSchema getOrCreatePendingRequest(final FetchDevice statement) { return pendingRequests.get(statement, k -> new FetchMissingDeviceSchema()); } - public void removeCompletedRequest(FetchDevice statement) { + public void removeCompletedRequest(final FetchDevice statement) { pendingRequests.invalidate(statement); } @@ -61,11 +61,11 @@ public Map> getResult() { return result; } - public void setResult(Map> result) { + public void setResult(final Map> result) { this.result = result; } - public synchronized void waitForQuery(long queryId) { + public synchronized void waitForQuery(final long queryId) { if (this.queryId != -1) { if (!done) { try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 9b0bea05bce9..59f66f0af6c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -144,7 +144,7 @@ Map> fetchMissingDeviceSchemaForDataInsertion( final List columnHeaderList = coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); - final int idLength = DataNodeTableCache.getInstance().getTable(database, table).getTagNum(); + final int tagLength = DataNodeTableCache.getInstance().getTable(database, table).getTagNum(); final Map> fetchedDeviceSchema = new HashMap<>(); while (coordinator.getQueryExecution(queryId).hasNextResult()) { @@ -166,7 +166,7 @@ Map> fetchMissingDeviceSchemaForDataInsertion( } final Column[] columns = tsBlock.get().getValueColumns(); for (int i = 0; i < tsBlock.get().getPositionCount(); i++) { - final String[] nodes = new String[idLength + 1]; + final String[] nodes = new String[tagLength + 1]; final Map attributeMap = new HashMap<>(); constructNodsArrayAndAttributeMap( attributeMap, nodes, table, columnHeaderList, columns, tableInstance, i); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java index 0a38501c0f23..cdb66fdc0474 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.db.exception.metadata.view.InsertNonWritableViewException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; @@ -30,7 +31,6 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java index 35c04bf4c185..09892f3bc407 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java @@ -230,6 +230,16 @@ TSStatus checkFullPathWriteDataPermission( TSStatus checkCanAlterView( IAuditEntity entity, List sourcePaths, List targetPaths); + TSStatus checkSeriesPrivilege4Pipe( + IAuditEntity context, + List checkedPathsSupplier, + PrivilegeType permission); + + List checkSeriesPrivilegeWithIndexes4Pipe( + IAuditEntity context, + List checkedPathsSupplier, + PrivilegeType permission); + // ====================================== COMMON ============================================= TSStatus allowUserToLogin(String userName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java index 4d720bc0ddcd..94323f30c96c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java @@ -563,6 +563,19 @@ public TSStatus checkCanAlterView( return status; } + @Override + public TSStatus checkSeriesPrivilege4Pipe( + IAuditEntity context, List checkedPaths, PrivilegeType permission) { + return TreeAccessCheckVisitor.checkTimeSeriesPermission( + context, () -> checkedPaths, permission); + } + + @Override + public List checkSeriesPrivilegeWithIndexes4Pipe( + IAuditEntity context, List checkedPaths, PrivilegeType permission) { + return TreeAccessCheckVisitor.checkTimeSeriesPermission4Pipe(context, checkedPaths, permission); + } + @Override public TSStatus allowUserToLogin(String userName) { return SUCCEED; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java index 11b20883dbbb..3d1d13944232 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java @@ -30,6 +30,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.function.Supplier; @@ -142,6 +143,22 @@ public TSStatus checkCanAlterView( return SUCCEED; } + @Override + public TSStatus checkSeriesPrivilege4Pipe( + IAuditEntity context, + List checkedPathsSupplier, + PrivilegeType permission) { + return SUCCEED; + } + + @Override + public List checkSeriesPrivilegeWithIndexes4Pipe( + IAuditEntity context, + List checkedPathsSupplier, + PrivilegeType permission) { + return Collections.emptyList(); + } + @Override public TSStatus allowUserToLogin(String userName) { return SUCCEED; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java index d939a85d7b45..384a565d9797 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java @@ -26,7 +26,9 @@ import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.db.audit.DNAuditLogger; import org.apache.iotdb.db.auth.AuthorityChecker; @@ -209,6 +211,33 @@ public TSStatus visitAuthorityInformation( return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } + public static List getIntersectedPaths4Pipe( + final List paths, final TreeAccessCheckContext context) { + context.setAuditLogOperation(AuditLogOperation.QUERY).setPrivilegeType(PrivilegeType.READ_DATA); + if (AuthorityChecker.SUPER_USER.equals(context.getUsername())) { + recordObjectAuthenticationAuditLog( + context.setResult(true), + () -> paths.stream().distinct().collect(Collectors.toList()).toString()); + return paths; + } + try { + final PathPatternTree originalTree = new PathPatternTree(); + paths.forEach(originalTree::appendPathPattern); + originalTree.constructTree(); + final PathPatternTree tree = + AuthorityChecker.getAuthorizedPathTree(context.getUsername(), PrivilegeType.READ_DATA); + recordObjectAuthenticationAuditLog( + context.setResult(true), + () -> paths.stream().distinct().collect(Collectors.toList()).toString()); + return originalTree.intersectWithFullPathPrefixTree(tree).getAllPathPatterns(true); + } catch (AuthException e) { + recordObjectAuthenticationAuditLog( + context.setResult(false), + () -> paths.stream().distinct().collect(Collectors.toList()).toString()); + return Collections.emptyList(); + } + } + // ====================== template related ================================= @Override @@ -935,7 +964,7 @@ public TSStatus visitAlterDatabase( } @Override - public TSStatus visitShowStorageGroup( + public TSStatus visitShowDatabase( ShowDatabaseStatement showDatabaseStatement, TreeAccessCheckContext context) { context .setAuditLogOperation(AuditLogOperation.QUERY) @@ -954,7 +983,7 @@ public TSStatus visitShowStorageGroup( } @Override - public TSStatus visitCountStorageGroup( + public TSStatus visitCountDatabase( CountDatabaseStatement countDatabaseStatement, TreeAccessCheckContext context) { context .setAuditLogOperation(AuditLogOperation.QUERY) @@ -973,7 +1002,7 @@ public TSStatus visitCountStorageGroup( } @Override - public TSStatus visitDeleteStorageGroup( + public TSStatus visitDeleteDatabase( DeleteDatabaseStatement statement, TreeAccessCheckContext context) { context .setAuditLogOperation(AuditLogOperation.DDL) @@ -1168,6 +1197,20 @@ public static TSStatus checkTimeSeriesPermission( return result; } + public static List checkTimeSeriesPermission4Pipe( + IAuditEntity context, List checkedPaths, PrivilegeType permission) { + context.setPrivilegeType(permission); + if (AuthorityChecker.SUPER_USER.equals(context.getUsername())) { + recordObjectAuthenticationAuditLog(context.setResult(true), checkedPaths::toString); + return Collections.emptyList(); + } + final List results = + AuthorityChecker.checkFullPathOrPatternListPermission( + context.getUsername(), checkedPaths, permission); + recordObjectAuthenticationAuditLog(context.setResult(true), checkedPaths::toString); + return results; + } + @Override public TSStatus visitCreateTimeseries( CreateTimeSeriesStatement statement, TreeAccessCheckContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipe.java index 3f53c5e4504d..11feffe7da10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipe.java @@ -29,24 +29,23 @@ public class CreatePipe extends PipeStatement { private final String pipeName; private final boolean ifNotExistsCondition; - private final Map extractorAttributes; + private final Map sourceAttributes; private final Map processorAttributes; - private final Map connectorAttributes; + private final Map sinkAttributes; public CreatePipe( final String pipeName, final boolean ifNotExistsCondition, - final Map extractorAttributes, + final Map sourceAttributes, final Map processorAttributes, - final Map connectorAttributes) { + final Map sinkAttributes) { this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); this.ifNotExistsCondition = ifNotExistsCondition; - this.extractorAttributes = - requireNonNull(extractorAttributes, "extractor/source attributes can not be null"); + this.sourceAttributes = + requireNonNull(sourceAttributes, "extractor/source attributes can not be null"); this.processorAttributes = requireNonNull(processorAttributes, "processor attributes can not be null"); - this.connectorAttributes = - requireNonNull(connectorAttributes, "connector attributes can not be null"); + this.sinkAttributes = requireNonNull(sinkAttributes, "connector attributes can not be null"); } public String getPipeName() { @@ -57,16 +56,16 @@ public boolean hasIfNotExistsCondition() { return ifNotExistsCondition; } - public Map getExtractorAttributes() { - return extractorAttributes; + public Map getSourceAttributes() { + return sourceAttributes; } public Map getProcessorAttributes() { return processorAttributes; } - public Map getConnectorAttributes() { - return connectorAttributes; + public Map getSinkAttributes() { + return sinkAttributes; } @Override @@ -77,11 +76,7 @@ public R accept(final AstVisitor visitor, final C context) { @Override public int hashCode() { return Objects.hash( - pipeName, - ifNotExistsCondition, - extractorAttributes, - processorAttributes, - connectorAttributes); + pipeName, ifNotExistsCondition, sourceAttributes, processorAttributes, sinkAttributes); } @Override @@ -95,9 +90,9 @@ public boolean equals(final Object obj) { CreatePipe other = (CreatePipe) obj; return Objects.equals(pipeName, other.pipeName) && Objects.equals(ifNotExistsCondition, other.ifNotExistsCondition) - && Objects.equals(extractorAttributes, other.extractorAttributes) + && Objects.equals(sourceAttributes, other.sourceAttributes) && Objects.equals(processorAttributes, other.processorAttributes) - && Objects.equals(connectorAttributes, other.connectorAttributes); + && Objects.equals(sinkAttributes, other.sinkAttributes); } @Override @@ -105,9 +100,9 @@ public String toString() { return toStringHelper(this) .add("pipeName", pipeName) .add("ifNotExistsCondition", ifNotExistsCondition) - .add("extractorAttributes", extractorAttributes) + .add("extractorAttributes", sourceAttributes) .add("processorAttributes", processorAttributes) - .add("connectorAttributes", connectorAttributes) + .add("connectorAttributes", sinkAttributes) .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index e428b6cb7832..0c1d862a886c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -1146,12 +1146,12 @@ protected Void visitCreatePipe(CreatePipe node, Integer context) { builder.append(node.getPipeName()); builder.append(" \n"); - if (!node.getExtractorAttributes().isEmpty()) { + if (!node.getSourceAttributes().isEmpty()) { builder .append("WITH SOURCE (") .append("\n") .append( - node.getExtractorAttributes().entrySet().stream() + node.getSourceAttributes().entrySet().stream() .map( entry -> indentString(1) @@ -1182,12 +1182,12 @@ protected Void visitCreatePipe(CreatePipe node, Integer context) { .append(")\n"); } - if (!node.getConnectorAttributes().isEmpty()) { + if (!node.getSinkAttributes().isEmpty()) { builder .append("WITH SINK (") .append("\n") .append( - node.getConnectorAttributes().entrySet().stream() + node.getSinkAttributes().entrySet().stream() .map( entry -> indentString(1) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 3bafdf8bfe0f..1dde7c5db3ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -200,7 +200,7 @@ public R visitDeleteTimeSeries(DeleteTimeSeriesStatement deleteTimeSeriesStateme return visitStatement(deleteTimeSeriesStatement, context); } - public R visitDeleteStorageGroup(DeleteDatabaseStatement deleteDatabaseStatement, C context) { + public R visitDeleteDatabase(DeleteDatabaseStatement deleteDatabaseStatement, C context) { return visitStatement(deleteDatabaseStatement, context); } @@ -384,7 +384,7 @@ public R visitAuthor(AuthorStatement authorStatement, C context) { return visitStatement(authorStatement, context); } - public R visitShowStorageGroup(ShowDatabaseStatement showDatabaseStatement, C context) { + public R visitShowDatabase(ShowDatabaseStatement showDatabaseStatement, C context) { return visitStatement(showDatabaseStatement, context); } @@ -396,7 +396,7 @@ public R visitShowDevices(ShowDevicesStatement showDevicesStatement, C context) return visitStatement(showDevicesStatement, context); } - public R visitCountStorageGroup(CountDatabaseStatement countDatabaseStatement, C context) { + public R visitCountDatabase(CountDatabaseStatement countDatabaseStatement, C context) { return visitStatement(countDatabaseStatement, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/InternalBatchActivateTemplateStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/InternalBatchActivateTemplateStatement.java index 7b7a19a9a8e2..48f3bf16b8bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/InternalBatchActivateTemplateStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/InternalBatchActivateTemplateStatement.java @@ -20,11 +20,11 @@ package org.apache.iotdb.db.queryengine.plan.statement.internal; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.utils.Pair; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java index 4380767da2e2..e41ce19fd482 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java @@ -21,10 +21,10 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.List; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDatabaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDatabaseStatement.java index 4ff683f44367..2d186ce41730 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDatabaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDatabaseStatement.java @@ -32,7 +32,7 @@ public CountDatabaseStatement(PartialPath partialPath) { @Override public R accept(StatementVisitor visitor, C context) { - return visitor.visitCountStorageGroup(this, context); + return visitor.visitCountDatabase(this, context); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DeleteDatabaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DeleteDatabaseStatement.java index 9861331974b3..6b538267555f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DeleteDatabaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DeleteDatabaseStatement.java @@ -63,7 +63,7 @@ public List getPrefixPath() { @Override public R accept(StatementVisitor visitor, C context) { - return visitor.visitDeleteStorageGroup(this, context); + return visitor.visitDeleteDatabase(this, context); } public void setPrefixPath(List prefixPathList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java index d91e1f65f0d8..e1a536622429 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java @@ -116,7 +116,7 @@ public void buildTSBlock( @Override public R accept(final StatementVisitor visitor, C context) { - return visitor.visitShowStorageGroup(this, context); + return visitor.visitShowDatabase(this, context); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java index 122139f3d4ea..a3738a15fbdf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java @@ -35,12 +35,12 @@ public class AlterPipeStatement extends Statement implements IConfigStatement { private String pipeName; private String userName; private boolean ifExistsCondition; - private Map extractorAttributes; + private Map sourceAttributes; private Map processorAttributes; - private Map connectorAttributes; - private boolean isReplaceAllExtractorAttributes; + private Map sinkAttributes; + private boolean isReplaceAllSourceAttributes; private boolean isReplaceAllProcessorAttributes; - private boolean isReplaceAllConnectorAttributes; + private boolean isReplaceAllSinkAttributes; private boolean isTableModel; public AlterPipeStatement(final StatementType alterPipeStatement) { @@ -55,28 +55,28 @@ public boolean hasIfExistsCondition() { return ifExistsCondition; } - public Map getExtractorAttributes() { - return extractorAttributes; + public Map getSourceAttributes() { + return sourceAttributes; } public Map getProcessorAttributes() { return processorAttributes; } - public Map getConnectorAttributes() { - return connectorAttributes; + public Map getSinkAttributes() { + return sinkAttributes; } - public boolean isReplaceAllExtractorAttributes() { - return isReplaceAllExtractorAttributes; + public boolean isReplaceAllSourceAttributes() { + return isReplaceAllSourceAttributes; } public boolean isReplaceAllProcessorAttributes() { return isReplaceAllProcessorAttributes; } - public boolean isReplaceAllConnectorAttributes() { - return isReplaceAllConnectorAttributes; + public boolean isReplaceAllSinkAttributes() { + return isReplaceAllSinkAttributes; } public boolean isTableModel() { @@ -95,28 +95,28 @@ public void setIfExists(final boolean ifExistsCondition) { this.ifExistsCondition = ifExistsCondition; } - public void setExtractorAttributes(final Map extractorAttributes) { - this.extractorAttributes = extractorAttributes; + public void setSourceAttributes(final Map sourceAttributes) { + this.sourceAttributes = sourceAttributes; } public void setProcessorAttributes(final Map processorAttributes) { this.processorAttributes = processorAttributes; } - public void setConnectorAttributes(final Map connectorAttributes) { - this.connectorAttributes = connectorAttributes; + public void setSinkAttributes(final Map sinkAttributes) { + this.sinkAttributes = sinkAttributes; } - public void setReplaceAllExtractorAttributes(final boolean replaceAllExtractorAttributes) { - isReplaceAllExtractorAttributes = replaceAllExtractorAttributes; + public void setReplaceAllSourceAttributes(final boolean replaceAllSourceAttributes) { + isReplaceAllSourceAttributes = replaceAllSourceAttributes; } public void setReplaceAllProcessorAttributes(final boolean replaceAllProcessorAttributes) { isReplaceAllProcessorAttributes = replaceAllProcessorAttributes; } - public void setReplaceAllConnectorAttributes(final boolean replaceAllConnectorAttributes) { - isReplaceAllConnectorAttributes = replaceAllConnectorAttributes; + public void setReplaceAllSinkAttributes(final boolean replaceAllConnectorAttributes) { + isReplaceAllSinkAttributes = replaceAllConnectorAttributes; } public void setTableModel(final boolean tableModel) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/CreatePipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/CreatePipeStatement.java index 66c7a85de79a..718283182a73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/CreatePipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/CreatePipeStatement.java @@ -34,11 +34,11 @@ public class CreatePipeStatement extends Statement implements IConfigStatement { private String pipeName; private boolean ifNotExistsCondition; - private Map extractorAttributes; + private Map sourceAttributes; private Map processorAttributes; - private Map connectorAttributes; + private Map sinkAttributes; - public CreatePipeStatement(StatementType createPipeStatement) { + public CreatePipeStatement(final StatementType createPipeStatement) { this.statementType = createPipeStatement; } @@ -50,19 +50,19 @@ public boolean hasIfNotExistsCondition() { return ifNotExistsCondition; } - public Map getExtractorAttributes() { - return extractorAttributes; + public Map getSourceAttributes() { + return sourceAttributes; } public Map getProcessorAttributes() { return processorAttributes; } - public Map getConnectorAttributes() { - return connectorAttributes; + public Map getSinkAttributes() { + return sinkAttributes; } - public void setPipeName(String pipeName) { + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } @@ -70,16 +70,16 @@ public void setIfNotExists(boolean ifNotExistsCondition) { this.ifNotExistsCondition = ifNotExistsCondition; } - public void setExtractorAttributes(Map extractorAttributes) { - this.extractorAttributes = extractorAttributes; + public void setSourceAttributes(final Map sourceAttributes) { + this.sourceAttributes = sourceAttributes; } - public void setProcessorAttributes(Map processorAttributes) { + public void setProcessorAttributes(final Map processorAttributes) { this.processorAttributes = processorAttributes; } - public void setConnectorAttributes(Map connectorAttributes) { - this.connectorAttributes = connectorAttributes; + public void setSinkAttributes(final Map sinkAttributes) { + this.sinkAttributes = sinkAttributes; } @Override @@ -93,7 +93,7 @@ public List getPaths() { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitCreatePipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/ActivateTemplateStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/ActivateTemplateStatement.java index e5abc9be3b33..67214c8bca16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/ActivateTemplateStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/ActivateTemplateStatement.java @@ -20,11 +20,11 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata.template; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.utils.Pair; @@ -49,13 +49,18 @@ public ActivateTemplateStatement(PartialPath path) { @Override public List getPaths() { + return getPaths(path); + } + + public static List getPaths(final PartialPath devicePath) { ClusterTemplateManager clusterTemplateManager = ClusterTemplateManager.getInstance(); - Pair templateSetInfo = clusterTemplateManager.checkTemplateSetInfo(path); + Pair templateSetInfo = + clusterTemplateManager.checkTemplateSetInfo(devicePath); if (templateSetInfo == null) { return Collections.emptyList(); } return templateSetInfo.left.getSchemaMap().keySet().stream() - .map(path::concatAsMeasurementPath) + .map(devicePath::concatAsMeasurementPath) .collect(Collectors.toList()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/BatchActivateTemplateStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/BatchActivateTemplateStatement.java index e518eee0d1ed..b92bc13abc86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/BatchActivateTemplateStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/BatchActivateTemplateStatement.java @@ -20,11 +20,11 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata.template; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.utils.Pair; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/DeactivateTemplateStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/DeactivateTemplateStatement.java index b2d7a5fef7b7..7057003d3945 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/DeactivateTemplateStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/template/DeactivateTemplateStatement.java @@ -21,13 +21,13 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.Collections; import java.util.List; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java index bd79953ee695..e5ab6e7ba95f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java @@ -19,8 +19,8 @@ package org.apache.iotdb.db.schemaengine.rescon; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.Map; import java.util.Objects; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index a9b002bbd6d1..0f96328112ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableId; @@ -54,7 +55,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.IRollbackPreDeactivateTemplatePlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.IAlterLogicalViewPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.ICreateLogicalViewPlan; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index fbb407d33388..a4a0758abb0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -31,6 +31,7 @@ import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.utils.FileUtils; @@ -122,7 +123,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.IPreDeleteLogicalViewPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.IRollbackPreDeleteLogicalViewPlan; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.utils.SchemaUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 0e180b2029b2..05adde04fac0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.schema.filter.SchemaFilterType; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -97,7 +98,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.IAlterLogicalViewPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.ICreateLogicalViewPlan; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.utils.SchemaUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/IMTreeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/IMTreeStore.java index 43286eafaddf..190c4794ff6f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/IMTreeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/IMTreeStore.java @@ -24,8 +24,8 @@ import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; -import org.apache.iotdb.db.schemaengine.template.Template; import java.io.File; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 387d8c664317..fc4fe6903a4c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -71,7 +72,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.impl.TimeseriesReaderWithViewFetch; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaFormatUtils; import org.apache.iotdb.db.schemaengine.schemaregion.utils.filter.DeviceFilterVisitor; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java index 08d03c29e69f..63a987b2cb7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.metric.SchemaRegionMemMetric; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; @@ -38,7 +39,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MNodeUtils; -import org.apache.iotdb.db.schemaengine.template.Template; import java.io.File; import java.io.IOException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java index 6f95102a26f1..5f72358418b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java @@ -24,9 +24,9 @@ import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MNodeUtils; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.Iterator; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MemoryTraverserIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MemoryTraverserIterator.java index 15eb95df4ab9..c222d5ecd866 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MemoryTraverserIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MemoryTraverserIterator.java @@ -22,8 +22,8 @@ import org.apache.iotdb.commons.schema.node.IMNode; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/CachedMTreeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/CachedMTreeStore.java index e15245d3c895..aa0108f9c1c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/CachedMTreeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/CachedMTreeStore.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.exception.metadata.cache.MNodeNotCachedException; import org.apache.iotdb.db.schemaengine.metric.SchemaRegionCachedMetric; import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaRegionStatistics; @@ -43,7 +44,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.schemafile.ISchemaFile; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MNodeUtils; -import org.apache.iotdb.db.schemaengine.template.Template; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java index d5187e14c914..58dacd73e8d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.exception.metadata.AliasAlreadyExistException; @@ -65,7 +66,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.impl.TimeseriesReaderWithViewFetch; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaFormatUtils; import org.apache.iotdb.db.schemaengine.schemaregion.utils.filter.DeviceFilterVisitor; -import org.apache.iotdb.db.schemaengine.template.Template; import com.google.common.util.concurrent.ListenableFuture; import org.apache.tsfile.enums.TSDataType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/ReentrantReadOnlyCachedMTreeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/ReentrantReadOnlyCachedMTreeStore.java index bffb93f59ac5..88b61c1b4c34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/ReentrantReadOnlyCachedMTreeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/ReentrantReadOnlyCachedMTreeStore.java @@ -23,10 +23,10 @@ import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode; -import org.apache.iotdb.db.schemaengine.template.Template; import java.io.File; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/iterator/CachedTraverserIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/iterator/CachedTraverserIterator.java index 0d14e2251c4d..529ddd44658d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/iterator/CachedTraverserIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/iterator/CachedTraverserIterator.java @@ -21,10 +21,10 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator.AbstractTraverserIterator; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java index 211f9e199991..24b7e93939e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java @@ -29,13 +29,13 @@ import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.tree.AbstractTreeVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator.MNodeIterator; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.ReentrantReadOnlyCachedMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MNodeUtils; -import org.apache.iotdb.db.schemaengine.template.Template; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/IShowTimeSeriesPlan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/IShowTimeSeriesPlan.java index c49280704467..e05ee54f253c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/IShowTimeSeriesPlan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/IShowTimeSeriesPlan.java @@ -21,7 +21,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.read.req; import org.apache.iotdb.commons.schema.filter.SchemaFilter; -import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.iotdb.commons.schema.template.Template; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/SchemaRegionReadPlanFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/SchemaRegionReadPlanFactory.java index eb0c2f64acce..84a2ca9cdf7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/SchemaRegionReadPlanFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/SchemaRegionReadPlanFactory.java @@ -22,10 +22,10 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowDevicesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowNodesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.impl.ShowTimeSeriesPlanImpl; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.Map; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTimeSeriesPlanImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTimeSeriesPlanImpl.java index ba9ea23cd3ae..8aeb0e837bed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTimeSeriesPlanImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/req/impl/ShowTimeSeriesPlanImpl.java @@ -23,8 +23,8 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowTimeSeriesPlan; -import org.apache.iotdb.db.schemaengine.template.Template; import java.util.Map; import java.util.Objects; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/MNodeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/MNodeUtils.java index 96fd00c848a0..6c5ac4b909d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/MNodeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/MNodeUtils.java @@ -22,8 +22,8 @@ import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.role.IInternalMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TreeDeviceInfo; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.write.schema.IMeasurementSchema; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/ClusterTemplateManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/ClusterTemplateManager.java index ac89c86b819d..857db6570773 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/ClusterTemplateManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/ClusterTemplateManager.java @@ -31,6 +31,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.confignode.rpc.thrift.TCreateSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/ITemplateManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/ITemplateManager.java index 3259e483ad03..e92866956bd5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/ITemplateManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/ITemplateManager.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.CreateSchemaTemplateStatement; import org.apache.tsfile.utils.Pair; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/TemplateInternalRPCUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/TemplateInternalRPCUtil.java index 72ae10f84294..0cec6abab0ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/TemplateInternalRPCUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/TemplateInternalRPCUtil.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.schemaengine.template; +import org.apache.iotdb.commons.schema.template.Template; + import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java index 9a6be9737af8..35e1e025bee7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java @@ -85,7 +85,7 @@ public Optional visitLoadTsFile( Long.MIN_VALUE, Long.MAX_VALUE, null, - "root", + null, null, true)) { for (final TabletInsertionEvent tabletInsertionEvent : parser.toTabletInsertionEvents()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java index b166e3d3772b..f21d1571b0f3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/TreeDeviceSchemaCacheManagerTest.java @@ -22,12 +22,12 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.SchemaCacheEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java index 7ad13612c691..4fae0f9fc0fc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -31,7 +32,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ISchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java index e51032f47d3f..aaed79515185 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.SessionInfo; @@ -44,7 +45,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateAlignedTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.isession.SessionConfig; import org.apache.tsfile.common.conf.TSFileConfig; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java index 4da6ce191e1a..f8590af08606 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; @@ -29,7 +30,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTestUtil.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTestUtil.java index 4f94de5d9565..42800c4e5863 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTestUtil.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTestUtil.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.filter.SchemaFilterFactory; import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; @@ -36,7 +37,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.impl.ShowTimeSeriesResult; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java index 61d641f757d9..441bf568cf86 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.node.IMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.schemaengine.SchemaEngine; import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaEngineStatistics; import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaRegionStatistics; @@ -34,7 +35,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java index 8516a9900e63..8a0dcb2f5973 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java @@ -264,7 +264,7 @@ private void createTablet() { } @Test - public void convertToTabletForTest() { + public void convertToTabletForTest() throws Exception { TabletInsertionEventTreePatternParser container1 = new TabletInsertionEventTreePatternParser(insertRowNode, new PrefixTreePattern(pattern)); Tablet tablet1 = container1.convertToTablet(); @@ -295,7 +295,7 @@ public void convertToTabletForTest() { } @Test - public void convertToAlignedTabletForTest() { + public void convertToAlignedTabletForTest() throws Exception { TabletInsertionEventTreePatternParser container1 = new TabletInsertionEventTreePatternParser( insertRowNodeAligned, new PrefixTreePattern(pattern)); @@ -328,13 +328,14 @@ public void convertToAlignedTabletForTest() { } @Test - public void convertToTabletWithFilteredRowsForTest() { + public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container1 = new TabletInsertionEventTreePatternParser( null, new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 111L, 113L), insertRowNode, - new PrefixTreePattern(pattern)); + new PrefixTreePattern(pattern), + null); Tablet tablet1 = container1.convertToTablet(); Assert.assertEquals(0, tablet1.getRowSize()); boolean isAligned1 = container1.isAligned(); @@ -345,7 +346,8 @@ public void convertToTabletWithFilteredRowsForTest() { null, new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 110L, 110L), insertRowNode, - new PrefixTreePattern(pattern)); + new PrefixTreePattern(pattern), + null); Tablet tablet2 = container2.convertToTablet(); Assert.assertEquals(1, tablet2.getRowSize()); boolean isAligned2 = container2.isAligned(); @@ -356,7 +358,8 @@ public void convertToTabletWithFilteredRowsForTest() { null, new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 111L, 113L), insertTabletNode, - new PrefixTreePattern(pattern)); + new PrefixTreePattern(pattern), + null); Tablet tablet3 = container3.convertToTablet(); Assert.assertEquals(3, tablet3.getRowSize()); boolean isAligned3 = container3.isAligned(); @@ -367,7 +370,8 @@ public void convertToTabletWithFilteredRowsForTest() { null, new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, Long.MIN_VALUE, 109L), insertTabletNode, - new PrefixTreePattern(pattern)); + new PrefixTreePattern(pattern), + null); Tablet tablet4 = container4.convertToTablet(); Assert.assertEquals(0, tablet4.getRowSize()); boolean isAligned4 = container4.isAligned(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java index 7bfde3b158d5..3af5028a676b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.pipe.event; -import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.PrefixTreePattern; @@ -36,7 +35,6 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TsFileSequenceReader; @@ -487,7 +485,7 @@ private void testToTabletInsertionEvents( testTsFilePointNum(nonalignedTsFile, notExistPattern, startTime, endTime, isQuery, 0); } - private void testMixedTsFileWithEmptyChunk(final boolean isQuery) throws IOException { + private void testMixedTsFileWithEmptyChunk(final boolean isQuery) throws Exception { final File tsFile = new File("0-0-1-0.tsfile"); resource = new TsFileResource(tsFile); resource.updatePlanIndexes(0); @@ -524,8 +522,7 @@ private void testMixedTsFileWithEmptyChunk(final boolean isQuery) throws IOExcep resource = null; } - private void testPartialNullValue(final boolean isQuery) - throws IOException, WriteProcessException, IllegalPathException { + private void testPartialNullValue(final boolean isQuery) throws Exception { alignedTsFile = new File("0-0-2-0.tsfile"); final List schemaList = new ArrayList<>(); @@ -592,7 +589,8 @@ private void testTsFilePointNum( tsFileContainer .toTabletInsertionEvents() .forEach( - event -> + event -> { + try { event .processRowByRow( (row, collector) -> { @@ -604,7 +602,8 @@ private void testTsFilePointNum( } }) .forEach( - tabletInsertionEvent1 -> + tabletInsertionEvent1 -> { + try { tabletInsertionEvent1 .processRowByRow( (row, collector) -> { @@ -616,7 +615,8 @@ private void testTsFilePointNum( } }) .forEach( - tabletInsertionEvent2 -> + tabletInsertionEvent2 -> { + try { tabletInsertionEvent2.processTablet( (tablet, rowCollector) -> new PipeRawTabletInsertionEvent(tablet, false) @@ -628,7 +628,19 @@ private void testTsFilePointNum( } catch (final IOException e) { throw new RuntimeException(e); } - }))))); + })); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); Assert.assertEquals(expectedCount, count1.get()); Assert.assertEquals(expectedCount, count2.get()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java index 4642c69721f1..b4a33d1bdd6e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.leaf.TimeSeriesViewOperand; import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaEntityNode; @@ -30,7 +31,6 @@ import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeVisitorFactory; import org.apache.iotdb.db.queryengine.common.schematree.visitor.SchemaTreeVisitorWithLimitOffsetWrapper; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java index ebc6e20a9bca..0aab35189457 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; @@ -30,7 +31,6 @@ import org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputationWithAutoCreation; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java index 7714bad18793..8e37fa5b6216 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java @@ -34,6 +34,7 @@ import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; @@ -54,7 +55,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; import org.apache.tsfile.enums.TSDataType; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java index 39fecf2d0e7c..040651d747b1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java @@ -33,6 +33,7 @@ import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; @@ -51,7 +52,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; import org.apache.tsfile.enums.TSDataType; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/statement/sys/pipe/PipeStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/statement/sys/pipe/PipeStatementTest.java index ab885ddb557d..04fccc195600 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/statement/sys/pipe/PipeStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/statement/sys/pipe/PipeStatementTest.java @@ -42,14 +42,14 @@ public void testCreatePipeStatement() { CreatePipeStatement statement = new CreatePipeStatement(StatementType.CREATE_PIPE); statement.setPipeName("test"); - statement.setExtractorAttributes(extractorAttributes); + statement.setSourceAttributes(extractorAttributes); statement.setProcessorAttributes(processorAttributes); - statement.setConnectorAttributes(connectorAttributes); + statement.setSinkAttributes(connectorAttributes); Assert.assertEquals("test", statement.getPipeName()); - Assert.assertEquals(extractorAttributes, statement.getExtractorAttributes()); + Assert.assertEquals(extractorAttributes, statement.getSourceAttributes()); Assert.assertEquals(processorAttributes, statement.getProcessorAttributes()); - Assert.assertEquals(connectorAttributes, statement.getConnectorAttributes()); + Assert.assertEquals(connectorAttributes, statement.getSinkAttributes()); Assert.assertEquals(QueryType.WRITE, statement.getQueryType()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java index ae26415f87b8..471a02c70b70 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.schemaRegion.SchemaRegionTestUtil; @@ -38,7 +39,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.ICreateAlignedTimeSeriesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.ICreateTimeSeriesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory; -import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.tools.schema.SRStatementGenerator; import org.apache.iotdb.db.tools.schema.SchemaRegionSnapshotParser; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java index 80e7df9ee5da..2612415682d9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/IoTDBTreePattern.java @@ -33,8 +33,12 @@ import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class IoTDBTreePattern extends SingleTreePattern { @@ -56,6 +60,17 @@ public IoTDBTreePattern(final String pattern) { this(true, pattern); } + public static List applyReversedIndexesOnList( + final List filteredIndexes, final List originalList) { + final Set indexes = new HashSet<>(filteredIndexes); + return Objects.nonNull(originalList) + ? IntStream.range(0, originalList.size()) + .filter(index -> !indexes.contains(index)) // 保留不在排除列表中的下标 + .mapToObj(originalList::get) + .collect(Collectors.toList()) + : null; + } + @Override public String getDefaultPattern() { return PipeSourceConstant.EXTRACTOR_PATTERN_IOTDB_DEFAULT_VALUE; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/Template.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/template/Template.java similarity index 99% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/Template.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/template/Template.java index e8c1c13dbae6..a2fcb33f5b88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/template/Template.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/template/Template.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.schemaengine.template; +package org.apache.iotdb.commons.schema.template; import org.apache.iotdb.commons.exception.IllegalPathException;