From 735db65d23e4f1042e70bd3a841308ef74b53b70 Mon Sep 17 00:00:00 2001 From: Socrates Date: Fri, 13 Dec 2024 09:56:53 +0800 Subject: [PATCH] [regression](paimon) test Paimon's connectivity of cloud filesystems (#45268) ### What problem does this PR solve? Problem Summary: Add some cases to test Paimon's connectivity of oss, obs, cos and cosn --- .../paimon/paimon_base_filesystem.out | 29 +++++---- .../paimon/paimon_base_filesystem.groovy | 65 +++++++++++++++---- 2 files changed, 68 insertions(+), 26 deletions(-) diff --git a/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out b/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out index d24ebd07ef6b16..6ce16ab2a38a2d 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out +++ b/regression-test/data/external_table_p0/paimon/paimon_base_filesystem.out @@ -1,17 +1,22 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !c1 -- -1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T14:35:38.768 -1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T15:58:54.364 +-- !oss -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 --- !c2 -- -7 Hugo -8 Stop +-- !obs -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 --- !c3 -- -1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T14:35:38.768 -1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T15:58:54.364 +-- !cos -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 --- !c4 -- -7 Hugo -8 Stop +-- !cosn -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !oss -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !obs -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !cos -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy index 1e4561c406556f..6ac76336986288 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy @@ -17,29 +17,48 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_docker_doris") { String enabled = context.config.otherConfigs.get("enablePaimonTest") - // if (enabled == null || !enabled.equalsIgnoreCase("true")) { - if (true) { - // temporary comment out, will add back when env is ready + + if (enabled == null || !enabled.equalsIgnoreCase("true")) { return } try { - String catalog_cos = "paimon_base_filesystem_paimon_cos" + String catalog_obs = "paimon_base_filesystem_paimon_obs" String catalog_oss = "paimon_base_filesystem_paimon_oss" + String catalog_cos = "paimon_base_filesystem_paimon_cos" + String catalog_cosn = "paimon_base_filesystem_paimon_cosn" String aliYunAk = context.config.otherConfigs.get("aliYunAk") String aliYunSk = context.config.otherConfigs.get("aliYunSk") + String hwYunAk = context.config.otherConfigs.get("hwYunAk") + String hwYunSk = context.config.otherConfigs.get("hwYunSk") String txYunAk = context.config.otherConfigs.get("txYunAk") String txYunSk = context.config.otherConfigs.get("txYunSk") - def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cos}.zd.all_table order by c18""" - def oss = """select * from ${catalog_oss}.paimonossdb1.test_tableoss order by a""" + def obs = """select * from ${catalog_obs}.db1.all_table limit 1;""" + def oss = """select * from ${catalog_oss}.db1.all_table limit 1;""" + def cos = """select * from ${catalog_cos}.db1.all_table limit 1;""" + def cosn = """select * from ${catalog_cosn}.db1.all_table limit 1;""" - sql """drop catalog if exists ${catalog_cos};""" + sql """drop catalog if exists ${catalog_obs};""" sql """drop catalog if exists ${catalog_oss};""" + sql """drop catalog if exists ${catalog_cos};""" + sql """drop catalog if exists ${catalog_cosn};""" + sql """ create catalog if not exists ${catalog_cos} properties ( "type" = "paimon", - "warehouse" = "cosn://doris-build-1308700295/regression/paimoncos", + "paimon.catalog.type"="filesystem", + "warehouse" = "s3://doris-build-1308700295/regression/paimon1", + "s3.access_key" = "${txYunAk}", + "s3.secret_key" = "${txYunSk}", + "s3.endpoint" = "cos.ap-beijing.myqcloud.com" + ); + """ + sql """ + create catalog if not exists ${catalog_cosn} properties ( + "type" = "paimon", + "paimon.catalog.type"="filesystem", + "warehouse" = "cosn://doris-build-1308700295/regression/paimon1", "cos.access_key" = "${txYunAk}", "cos.secret_key" = "${txYunSk}", "cos.endpoint" = "cos.ap-beijing.myqcloud.com" @@ -48,22 +67,40 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock sql """ create catalog if not exists ${catalog_oss} properties ( "type" = "paimon", - "warehouse" = "oss://doris-regression-bj/regression/paimonoss", + "paimon.catalog.type"="filesystem", + "warehouse" = "oss://doris-regression-bj/regression/paimon1", "oss.access_key"="${aliYunAk}", "oss.secret_key"="${aliYunSk}", "oss.endpoint"="oss-cn-beijing.aliyuncs.com" ); """ - logger.info("catalog " + catalog_cos + " created") + sql """ + create catalog if not exists ${catalog_obs} properties ( + "type" = "paimon", + "paimon.catalog.type"="filesystem", + "warehouse" = "obs://doris-build/regression/paimon1", + "obs.access_key"="${hwYunAk}", + "obs.secret_key"="${hwYunSk}", + "obs.endpoint"="obs.cn-north-4.myhuaweicloud.com" + ); + """ + logger.info("catalog " + catalog_obs + " created") logger.info("catalog " + catalog_oss + " created") + logger.info("catalog " + catalog_cos + " created") + logger.info("catalog " + catalog_cosn + " created") sql """set force_jni_scanner=false""" - qt_c1 cos - qt_c2 oss + qt_oss oss + qt_obs obs + qt_cos cos + qt_cosn cosn sql """set force_jni_scanner=true""" - qt_c3 cos - qt_c4 oss + qt_oss oss + qt_obs obs + qt_cos cos + // java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.CosFileSystem not found + // qt_cosn cosn } finally { sql """set force_jni_scanner=false"""