Skip to content

Commit

Permalink
[regression](paimon) test Paimon's connectivity of cloud filesystems (a…
Browse files Browse the repository at this point in the history
…pache#45268)

### What problem does this PR solve?

Problem Summary:
Add some cases to test Paimon's connectivity of oss, obs, cos and cosn
  • Loading branch information
suxiaogang223 authored Dec 13, 2024
1 parent c0f4c89 commit 735db65
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 26 deletions.
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !c1 --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T14:35:38.768
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T15:58:54.364
-- !oss --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !c2 --
7 Hugo
8 Stop
-- !obs --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !c3 --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T14:35:38.768
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true 2023-08-10T15:58:54.364
-- !cos --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !c4 --
7 Hugo
8 Stop
-- !cosn --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !oss --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !obs --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

-- !cos --
1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530

Original file line number Diff line number Diff line change
Expand Up @@ -17,29 +17,48 @@

suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_docker_doris") {
String enabled = context.config.otherConfigs.get("enablePaimonTest")
// if (enabled == null || !enabled.equalsIgnoreCase("true")) {
if (true) {
// temporary comment out, will add back when env is ready

if (enabled == null || !enabled.equalsIgnoreCase("true")) {
return
}

try {
String catalog_cos = "paimon_base_filesystem_paimon_cos"
String catalog_obs = "paimon_base_filesystem_paimon_obs"
String catalog_oss = "paimon_base_filesystem_paimon_oss"
String catalog_cos = "paimon_base_filesystem_paimon_cos"
String catalog_cosn = "paimon_base_filesystem_paimon_cosn"
String aliYunAk = context.config.otherConfigs.get("aliYunAk")
String aliYunSk = context.config.otherConfigs.get("aliYunSk")
String hwYunAk = context.config.otherConfigs.get("hwYunAk")
String hwYunSk = context.config.otherConfigs.get("hwYunSk")
String txYunAk = context.config.otherConfigs.get("txYunAk")
String txYunSk = context.config.otherConfigs.get("txYunSk")

def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cos}.zd.all_table order by c18"""
def oss = """select * from ${catalog_oss}.paimonossdb1.test_tableoss order by a"""
def obs = """select * from ${catalog_obs}.db1.all_table limit 1;"""
def oss = """select * from ${catalog_oss}.db1.all_table limit 1;"""
def cos = """select * from ${catalog_cos}.db1.all_table limit 1;"""
def cosn = """select * from ${catalog_cosn}.db1.all_table limit 1;"""

sql """drop catalog if exists ${catalog_cos};"""
sql """drop catalog if exists ${catalog_obs};"""
sql """drop catalog if exists ${catalog_oss};"""
sql """drop catalog if exists ${catalog_cos};"""
sql """drop catalog if exists ${catalog_cosn};"""

sql """
create catalog if not exists ${catalog_cos} properties (
"type" = "paimon",
"warehouse" = "cosn://doris-build-1308700295/regression/paimoncos",
"paimon.catalog.type"="filesystem",
"warehouse" = "s3://doris-build-1308700295/regression/paimon1",
"s3.access_key" = "${txYunAk}",
"s3.secret_key" = "${txYunSk}",
"s3.endpoint" = "cos.ap-beijing.myqcloud.com"
);
"""
sql """
create catalog if not exists ${catalog_cosn} properties (
"type" = "paimon",
"paimon.catalog.type"="filesystem",
"warehouse" = "cosn://doris-build-1308700295/regression/paimon1",
"cos.access_key" = "${txYunAk}",
"cos.secret_key" = "${txYunSk}",
"cos.endpoint" = "cos.ap-beijing.myqcloud.com"
Expand All @@ -48,22 +67,40 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock
sql """
create catalog if not exists ${catalog_oss} properties (
"type" = "paimon",
"warehouse" = "oss://doris-regression-bj/regression/paimonoss",
"paimon.catalog.type"="filesystem",
"warehouse" = "oss://doris-regression-bj/regression/paimon1",
"oss.access_key"="${aliYunAk}",
"oss.secret_key"="${aliYunSk}",
"oss.endpoint"="oss-cn-beijing.aliyuncs.com"
);
"""
logger.info("catalog " + catalog_cos + " created")
sql """
create catalog if not exists ${catalog_obs} properties (
"type" = "paimon",
"paimon.catalog.type"="filesystem",
"warehouse" = "obs://doris-build/regression/paimon1",
"obs.access_key"="${hwYunAk}",
"obs.secret_key"="${hwYunSk}",
"obs.endpoint"="obs.cn-north-4.myhuaweicloud.com"
);
"""
logger.info("catalog " + catalog_obs + " created")
logger.info("catalog " + catalog_oss + " created")
logger.info("catalog " + catalog_cos + " created")
logger.info("catalog " + catalog_cosn + " created")

sql """set force_jni_scanner=false"""
qt_c1 cos
qt_c2 oss
qt_oss oss
qt_obs obs
qt_cos cos
qt_cosn cosn

sql """set force_jni_scanner=true"""
qt_c3 cos
qt_c4 oss
qt_oss oss
qt_obs obs
qt_cos cos
// java.lang.ClassNotFoundException: Class org.apache.hadoop.fs.CosFileSystem not found
// qt_cosn cosn

} finally {
sql """set force_jni_scanner=false"""
Expand Down

0 comments on commit 735db65

Please sign in to comment.