diff --git a/common/src/main/java/com/dremio/io/file/UriSchemes.java b/common/src/main/java/com/dremio/io/file/UriSchemes.java
index 24514e81d5..c5209dd8f8 100644
--- a/common/src/main/java/com/dremio/io/file/UriSchemes.java
+++ b/common/src/main/java/com/dremio/io/file/UriSchemes.java
@@ -24,6 +24,7 @@ public interface UriSchemes {
String WEBHDFS_SCHEME = "webhdfs";
String FILE_SCHEME = "file";
String S3_SCHEME = "s3";
+ String COS_SCHEME = "cosn";
String AZURE_SCHEME = "wasbs";
String GCS_SCHEME = "gs";
String ADL_SCHEME = "adl";
diff --git a/plugins/s3/pom.xml b/plugins/s3/pom.xml
index 320cec92db..9fefb0a6cb 100644
--- a/plugins/s3/pom.xml
+++ b/plugins/s3/pom.xml
@@ -34,6 +34,14 @@
jar
+
+ com.qcloud.cos
+ hadoop-cos
+
+
+ com.qcloud
+ cos_api-bundle
+
com.amazonaws
aws-java-sdk-s3
diff --git a/pom.xml b/pom.xml
index b37cff6398..25ff29d519 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1600,6 +1600,16 @@ limitations under the License.
+
+ com.qcloud.cos
+ hadoop-cos
+ 3.1.0-5.9.3
+
+
+ com.qcloud
+ cos_api-bundle
+ 5.6.35
+
commons-beanutils
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/hadoop/DremioHadoopUtils.java b/sabot/kernel/src/main/java/com/dremio/exec/hadoop/DremioHadoopUtils.java
index a627523c21..db387116ef 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/hadoop/DremioHadoopUtils.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/hadoop/DremioHadoopUtils.java
@@ -31,6 +31,8 @@
*/
public class DremioHadoopUtils {
+ public static final String COS_SCHEME = "cosn";
+
public static String getHadoopFSScheme(Path path, Configuration conf) {
return Util.getFs(path, conf).getScheme();
}
@@ -50,6 +52,9 @@ public static Path toHadoopPath(String path) {
* @return container name
*/
public static String getContainerName(Path path) {
+ if (COS_SCHEME.equalsIgnoreCase(path.toUri().getScheme())) {
+ return path.toString().split(Path.SEPARATOR)[2];
+ }
final List pathComponents = Arrays.asList(
removeLeadingSlash(Path.getPathWithoutSchemeAndAuthority(path).toString())
.split(Path.SEPARATOR)
@@ -59,6 +64,9 @@ public static String getContainerName(Path path) {
public static Path pathWithoutContainer(Path path) {
List pathComponents = Arrays.asList(removeLeadingSlash(Path.getPathWithoutSchemeAndAuthority(path).toString()).split(Path.SEPARATOR));
+ if (COS_SCHEME.equalsIgnoreCase(path.toUri().getScheme())) {
+ return new Path("/" + Joiner.on(Path.SEPARATOR).join(pathComponents));
+ }
return new Path("/" + Joiner.on(Path.SEPARATOR).join(pathComponents.subList(1, pathComponents.size())));
}
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/iceberg/IcebergUtils.java b/sabot/kernel/src/main/java/com/dremio/exec/store/iceberg/IcebergUtils.java
index 3614d74ce6..cbab8efc40 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/iceberg/IcebergUtils.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/iceberg/IcebergUtils.java
@@ -23,6 +23,7 @@
import static com.dremio.io.file.Path.SEPARATOR;
import static com.dremio.io.file.UriSchemes.ADL_SCHEME;
import static com.dremio.io.file.UriSchemes.AZURE_SCHEME;
+import static com.dremio.io.file.UriSchemes.COS_SCHEME;
import static com.dremio.io.file.UriSchemes.FILE_SCHEME;
import static com.dremio.io.file.UriSchemes.GCS_SCHEME;
import static com.dremio.io.file.UriSchemes.HDFS_SCHEME;
@@ -385,12 +386,14 @@ public static String getValidIcebergPath(Path path, Configuration conf, String f
urlBuilder.append(SCHEME_SEPARATOR);
urlBuilder.append(getContainerName(path));
urlBuilder.append(CONTAINER_SEPARATOR + accountName + AZURE_AUTHORITY_SUFFIX);
- urlBuilder.append(pathWithoutContainer(path).toString());
+ urlBuilder.append(pathWithoutContainer(path));
return urlBuilder.toString();
} else if (fsScheme.equalsIgnoreCase(FileSystemConf.CloudFileSystemScheme.S3_FILE_SYSTEM_SCHEME.getScheme())) {
return S3_SCHEME + SCHEME_SEPARATOR + modifiedPath;
} else if (fsScheme.equalsIgnoreCase(FileSystemConf.CloudFileSystemScheme.GOOGLE_CLOUD_FILE_SYSTEM.getScheme())) {
return GCS_SCHEME + SCHEME_SEPARATOR + modifiedPath;
+ } else if (fsScheme.equalsIgnoreCase(COS_SCHEME)) {
+ return COS_SCHEME + SCHEME_SEPARATOR + modifiedPath;
} else if (fsScheme.equalsIgnoreCase(HDFS_SCHEME)) {
String hdfsEndPoint = conf.get("fs.defaultFS");
if (hdfsEndPoint == null || !hdfsEndPoint.toLowerCase().startsWith(HDFS_SCHEME)) {
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetSplitReaderCreator.java b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetSplitReaderCreator.java
index 322e42eeab..7810b69daf 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetSplitReaderCreator.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetSplitReaderCreator.java
@@ -136,7 +136,7 @@ public ParquetSplitReaderCreator(boolean autoCorrectCorruptDates,
this.splitXAttr = splitXAttr;
this.path = Path.of(splitXAttr.getPath());
this.tablePath = tablePath;
- if (!fs.supportsPath(path)) {
+ if (!"cosn".equalsIgnoreCase(path.toURI().getScheme()) && !fs.supportsPath(path)) {
throw UserException.invalidMetadataError()
.addContext(String.format("%s: Invalid FS for file '%s'", fs.getScheme(), path))
.addContext("File", path)