test: add integration test with minio

.github/workflows/ci.yml

@@ -70,6 +70,19 @@ jobs:
           name: cov-report-rust-tests-${{ runner.os }}
           path: ./cov-reports
           if-no-files-found: 'error'
+
+  rust-integration-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-22.04 ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Spin up minio for s3 tests
+        run: docker compose -f docker/docker-compose.yaml --profile s3 up -d
+      - name: Run Rust integration tests
+        run: docker compose -f docker/docker-compose.yaml run integration_test_rust
 
   python-tests:
     strategy:

.gitignore

@@ -16,6 +16,7 @@
 # under the License.
 
 /Cargo.lock
+**/Cargo.lock
 /target
 **/target
 

crates/tests/src/lib.rs

@@ -61,6 +61,13 @@
         path_buf.to_str().unwrap().to_string()
     }
 
+    pub fn s3_path(&self) -> String {
+        let bucket = std::env::var("INTEGRATION_TEST_S3_BUCKET")
+            .expect("INTEGRATION_TEST_S3_BUCKET not set");
+        let data_path = Path::new(format!("s3://{}", bucket).as_str()).join(self.as_ref());
+        data_path.to_str().unwrap().to_string()
+    }
+
     pub fn url(&self) -> Url {
         let path = self.path();
         Url::from_file_path(path).unwrap()

docker/Dockerfile.minio

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM alpine:3.18
+RUN apk update && apk add --no-cache wget ca-certificates bash findutils
+
+RUN cd /usr/local/bin && \
+    wget -q --show-progress https://dl.min.io/client/mc/release/linux-amd64/mc && \
+    chmod +x mc

docker/docker-compose.yaml

@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: "3.8"
+
+services:
+  minio:
+    image: quay.io/minio/minio
+    container_name: minio
+    profiles:
+      - s3
+    ports:
+      - "9000:9000"
+    networks:
+      - hudi_network
+    env_file:
+      - hudi.env
+    command: server /data
+    healthcheck:
+      test: mc ready local
+      interval: 5s
+      timeout: 10s
+      retries: 5
+      start_period: 2s
+
+
+  mc:
+    build:
+      context: ./
+      dockerfile: ./Dockerfile.minio
+    container_name: mc
+    networks:
+      - hudi_network
+    profiles:
+      - s3
+    depends_on:
+      minio:
+        condition: service_healthy
+    env_file:
+      - hudi.env
+    volumes:
+      - ../crates/tests/data/tables:/opt/data:ro
+    entrypoint: >
+      /bin/bash -c '
+        set -eux
+        mc config host add store http://minio:9000 $$MINIO_ACCESS_KEY $$MINIO_SECRET_KEY
+        mc mb store/$$MINIO_TEST_BUCKET
+        find /opt/data -name "*.zip" -exec unzip {} -d /tmp/data/ \;
+        find /tmp/data -type d -mindepth 1 -maxdepth 1 -exec mc cp --recursive {} store/$$MINIO_TEST_BUCKET \;
+        mc ls store/$$MINIO_TEST_BUCKET
+      '
+
+  integration_test_rust:
+    image: rust:1.75
+    container_name: integration_test_rust
+    networks:
+      - hudi_network
+    profiles:
+      - integration_test
+    env_file:
+      - hudi.env
+    volumes:
+      - ../crates:/app/crates:ro
+      - ../python:/app/python:ro
+      - ../integration_test:/app/integration_test
+      - ../Cargo.toml:/app/Cargo.toml:ro
+    entrypoint: >
+      /bin/sh -c "
+        set -eux
+        apt update && apt install -y --no-install-recommends libpython3.11-dev
+        cd /app/integration_test
+        cargo test
+      "
+
+
+networks:
+  hudi_network:
+    driver: bridge

docker/hudi.env

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# setup minio
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=minio123
+MINIO_ACCESS_KEY=minio
+MINIO_SECRET_KEY=minio123
+MINIO_TEST_BUCKET=test-bucket
+
+# s3 integration test
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+AWS_ENDPOINT=http://minio:9000
+INTEGRATION_TEST_S3_BUCKET=test-bucket

integration_test/Cargo.toml

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[workspace]
+members = []
+
+[package]
+version = "0.1.0"
+name = "integration_test"
+edition = "2021"
+license = "Apache-2.0"
+rust-version = "1.75.0"
+description = "Integration tests for hudi-rs"
+
+
+[dependencies]
+hudi = { features=["datafusion"], path="../crates/hudi" }
+hudi-tests = { path="../crates/tests" }
+datafusion = { version = "= 39.0.0" }
+datafusion-expr = { version = "= 39.0.0" }
+tokio = { version = "1", features = ["rt-multi-thread"] }

integration_test/src/main.rs

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+fn main() {
+    println!("hudi-rs integration tests")
+}