Skip to content

Commit 8dcb19d

Browse files
ArmandoZArmandoZ
committed
ci: Add test case for hdfs over gcs bucket (#3504)
1 parent d324fb2 commit 8dcb19d

File tree

3 files changed

+120
-0
lines changed

3 files changed

+120
-0
lines changed

‎.github/services/hdfs/hdfs_default_gcs/action.yml

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: hdfs_default_gcs
19+
description: 'Behavior test for hdfs default over gcs'
20+
21+
runs:
22+
using: "composite"
23+
steps:
24+
- name: Setup java env
25+
uses: actions/setup-java@v4
26+
with:
27+
distribution: temurin
28+
java-version: "11"
29+
- name: Load secrets
30+
uses: 1password/load-secrets-action@v1
31+
with:
32+
export-env: true
33+
env:
34+
OPENDAL_GCS_ROOT: op://services/gcs/root
35+
OPENDAL_GCS_BUCKET: op://services/gcs/bucket
36+
OPENDAL_GCS_CREDENTIAL: op://services/gcs/credential
37+
- name: Setup
38+
shell: bash
39+
run: |
40+
curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
41+
42+
export HADOOP_HOME="/home/runner/hadoop-3.3.5"
43+
export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
44+
45+
curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/gcs-connector-hadoop3-2.2.19-shaded.jar https://github.com/GoogleCloudDataproc/hadoop-connectors/releases/download/v2.2.19/gcs-connector-hadoop3-2.2.19-shaded.jar
46+
47+
cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
48+
cp ./fixtures/hdfs/gcs-core-site.xml ${HADOOP_HOME}/etc/hadoop/core-site.xml
49+
50+
cat << EOF >> $GITHUB_ENV
51+
HADOOP_HOME=${HADOOP_HOME}
52+
CLASSPATH=${CLASSPATH}
53+
LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
54+
OPENDAL_HDFS_ROOT=${OPENDAL_GCS_ROOT}
55+
OPENDAL_HDFS_NAME_NODE=gs://${OPENDAL_GCS_BUCKET}
56+
OPENDAL_HDFS_ENABLE_APPEND=false
57+
EOF
58+
59+
mkdir -p /tmp/hdfs
60+
61+
cat << EOF > /tmp/hdfs/gcs-credentials.json
62+
`echo ${OPENDAL_GCS_CREDENTIAL} | base64 -d`
63+
EOF
64+

‎core/src/services/hdfs/docs.md

+1
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ async fn main() -> Result<()> {
121121
// Create fs backend builder.
122122
let mut builder = Hdfs::default();
123123
// Set the name node for hdfs.
124+
// If the string starts with a protocol type such as file://, hdfs://, or gs://, this protocol type will be used.
124125
builder.name_node("hdfs://127.0.0.1:9000");
125126
// Set the root for hdfs, all operations will happen under this root.
126127
//

‎fixtures/hdfs/gcs-core-site.xml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<!--
2+
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
20+
-->
21+
<?xml version="1.0" encoding="UTF-8"?>
22+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
23+
24+
<!-- Put site-specific property overrides in this file. -->
25+
26+
<configuration>
27+
<property>
28+
<name>fs.AbstractFileSystem.gs.impl</name>
29+
<value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</value>
30+
<description>The AbstractFileSystem for 'gs:' URIs.</description>
31+
</property>
32+
<property>
33+
<name>fs.gs.project.id</name>
34+
<value></value>
35+
<description>
36+
Optional. Google Cloud Project ID with access to GCS buckets.
37+
Required only for list buckets and create bucket operations.
38+
</description>
39+
</property>
40+
<property>
41+
<name>google.cloud.auth.type</name>
42+
<value>SERVICE_ACCOUNT_JSON_KEYFILE</value>
43+
<description>
44+
Authentication type to use for GCS access.
45+
</description>
46+
</property>
47+
<property>
48+
<name>google.cloud.auth.service.account.json.keyfile</name>
49+
<value>/tmp/hdfs/gcs-credentials.json</value>
50+
<description>
51+
The JSON keyfile of the service account used for GCS
52+
access when google.cloud.auth.type is SERVICE_ACCOUNT_JSON_KEYFILE.
53+
</description>
54+
</property>
55+
</configuration>

0 commit comments

Comments
 (0)