diff --git a/common/src/main/java/com/dremio/io/file/UriSchemes.java b/common/src/main/java/com/dremio/io/file/UriSchemes.java index 24514e81d5..c5209dd8f8 100644 --- a/common/src/main/java/com/dremio/io/file/UriSchemes.java +++ b/common/src/main/java/com/dremio/io/file/UriSchemes.java @@ -24,6 +24,7 @@ public interface UriSchemes { String WEBHDFS_SCHEME = "webhdfs"; String FILE_SCHEME = "file"; String S3_SCHEME = "s3"; + String COS_SCHEME = "cosn"; String AZURE_SCHEME = "wasbs"; String GCS_SCHEME = "gs"; String ADL_SCHEME = "adl"; diff --git a/plugins/s3/pom.xml b/plugins/s3/pom.xml index 320cec92db..9fefb0a6cb 100644 --- a/plugins/s3/pom.xml +++ b/plugins/s3/pom.xml @@ -34,6 +34,14 @@ jar + + com.qcloud.cos + hadoop-cos + + + com.qcloud + cos_api-bundle + com.amazonaws aws-java-sdk-s3 diff --git a/pom.xml b/pom.xml index b37cff6398..25ff29d519 100755 --- a/pom.xml +++ b/pom.xml @@ -1600,6 +1600,16 @@ limitations under the License. + + com.qcloud.cos + hadoop-cos + 3.1.0-5.9.3 + + + com.qcloud + cos_api-bundle + 5.6.35 + commons-beanutils diff --git a/sabot/kernel/src/main/java/com/dremio/exec/hadoop/DremioHadoopUtils.java b/sabot/kernel/src/main/java/com/dremio/exec/hadoop/DremioHadoopUtils.java index a627523c21..db387116ef 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/hadoop/DremioHadoopUtils.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/hadoop/DremioHadoopUtils.java @@ -31,6 +31,8 @@ */ public class DremioHadoopUtils { + public static final String COS_SCHEME = "cosn"; + public static String getHadoopFSScheme(Path path, Configuration conf) { return Util.getFs(path, conf).getScheme(); } @@ -50,6 +52,9 @@ public static Path toHadoopPath(String path) { * @return container name */ public static String getContainerName(Path path) { + if (COS_SCHEME.equalsIgnoreCase(path.toUri().getScheme())) { + return path.toString().split(Path.SEPARATOR)[2]; + } final List pathComponents = Arrays.asList( removeLeadingSlash(Path.getPathWithoutSchemeAndAuthority(path).toString()) .split(Path.SEPARATOR) @@ -59,6 +64,9 @@ public static String getContainerName(Path path) { public static Path pathWithoutContainer(Path path) { List pathComponents = Arrays.asList(removeLeadingSlash(Path.getPathWithoutSchemeAndAuthority(path).toString()).split(Path.SEPARATOR)); + if (COS_SCHEME.equalsIgnoreCase(path.toUri().getScheme())) { + return new Path("/" + Joiner.on(Path.SEPARATOR).join(pathComponents)); + } return new Path("/" + Joiner.on(Path.SEPARATOR).join(pathComponents.subList(1, pathComponents.size()))); } } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/iceberg/IcebergUtils.java b/sabot/kernel/src/main/java/com/dremio/exec/store/iceberg/IcebergUtils.java index 3614d74ce6..cbab8efc40 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/iceberg/IcebergUtils.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/iceberg/IcebergUtils.java @@ -23,6 +23,7 @@ import static com.dremio.io.file.Path.SEPARATOR; import static com.dremio.io.file.UriSchemes.ADL_SCHEME; import static com.dremio.io.file.UriSchemes.AZURE_SCHEME; +import static com.dremio.io.file.UriSchemes.COS_SCHEME; import static com.dremio.io.file.UriSchemes.FILE_SCHEME; import static com.dremio.io.file.UriSchemes.GCS_SCHEME; import static com.dremio.io.file.UriSchemes.HDFS_SCHEME; @@ -385,12 +386,14 @@ public static String getValidIcebergPath(Path path, Configuration conf, String f urlBuilder.append(SCHEME_SEPARATOR); urlBuilder.append(getContainerName(path)); urlBuilder.append(CONTAINER_SEPARATOR + accountName + AZURE_AUTHORITY_SUFFIX); - urlBuilder.append(pathWithoutContainer(path).toString()); + urlBuilder.append(pathWithoutContainer(path)); return urlBuilder.toString(); } else if (fsScheme.equalsIgnoreCase(FileSystemConf.CloudFileSystemScheme.S3_FILE_SYSTEM_SCHEME.getScheme())) { return S3_SCHEME + SCHEME_SEPARATOR + modifiedPath; } else if (fsScheme.equalsIgnoreCase(FileSystemConf.CloudFileSystemScheme.GOOGLE_CLOUD_FILE_SYSTEM.getScheme())) { return GCS_SCHEME + SCHEME_SEPARATOR + modifiedPath; + } else if (fsScheme.equalsIgnoreCase(COS_SCHEME)) { + return COS_SCHEME + SCHEME_SEPARATOR + modifiedPath; } else if (fsScheme.equalsIgnoreCase(HDFS_SCHEME)) { String hdfsEndPoint = conf.get("fs.defaultFS"); if (hdfsEndPoint == null || !hdfsEndPoint.toLowerCase().startsWith(HDFS_SCHEME)) { diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetSplitReaderCreator.java b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetSplitReaderCreator.java index 322e42eeab..7810b69daf 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetSplitReaderCreator.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetSplitReaderCreator.java @@ -136,7 +136,7 @@ public ParquetSplitReaderCreator(boolean autoCorrectCorruptDates, this.splitXAttr = splitXAttr; this.path = Path.of(splitXAttr.getPath()); this.tablePath = tablePath; - if (!fs.supportsPath(path)) { + if (!"cosn".equalsIgnoreCase(path.toURI().getScheme()) && !fs.supportsPath(path)) { throw UserException.invalidMetadataError() .addContext(String.format("%s: Invalid FS for file '%s'", fs.getScheme(), path)) .addContext("File", path)