Feat/core/replace rowstream with arrow tpch

.github/actions/dependence/action.yml

@@ -83,6 +83,7 @@ runs:
               echo "tqdm" >> requirements.txt
               echo "requests" >> requirements.txt
               echo "torch" >> requirements.txt
+              echo "pyarrow" >> requirements.txt
 
         - if: inputs.scope=='all' && inputs.iginx-conda-flag != 'true'
           name: Set up Python ${{ inputs.python-version }}

.github/actions/service/freeThreadConda/action.yml

@@ -27,10 +27,12 @@ runs:
               python -VV
               python -c "import sys;print(sys._is_gil_enabled())"
               curl -L -O https://github.com/IGinX-THU/IGinX-resources/raw/refs/heads/main/resources/python/pandas-3.0.0.dev0+1654.g32a97a969a-cp313-cp313t-win_amd64.whl
+              curl -L -O https://pypi.fury.io/arrow-nightlies/-/ver_1AsSYu/pyarrow-19.0.0.dev286-cp313-cp313t-win_amd64.whl#md5=3c940f7b4ffd0debaa8b23d06bc9b2ea
               ls -l
               if [ "$RUNNER_OS" == "Windows" ]; then
                python -m pip install numpy thrift pemjax
                python -m pip install pandas*.whl
+               python3.13t -m pip install pyarrow*.whl
               else
                python -m pip install pandas==2.2.3 numpy thrift pemjax
               fi

.github/actions/tpchSingleTest/action.yml

@@ -49,17 +49,26 @@ runs:
         - name: Run SQL Warmup on Old IGinX
           if: inputs.status != 'ok'
           shell: bash -el {0}
+          working-directory: IGinX
           run: mvn test -q -Dtest=SQLWarmupIT -DfailIfNoTests=false -P-format
 
         - name: Run TPCH Test on Old IGinX
           if: inputs.status != 'ok'
           shell: bash -el {0}
-          run: mvn test -q -Dtest=TPCHOldIT -DfailIfNoTests=false -P-format
+          working-directory: IGinX
+          run: |
+              mkdir -p test/src/test/resources/tpch/runtimeInfo
+              cp -f ../test/src/test/resources/tpch/runtimeInfo/newTimeCosts.txt test/src/test/resources/tpch/runtimeInfo/newTimeCosts.txt
+              mvn test -q -Dtest=TPCHOldIT -DfailIfNoTests=false -P-format
+              mkdir -p ../test/src/test/resources/tpch/runtimeInfo
+              cp -f test/src/test/resources/tpch/runtimeInfo/failedQueryIds.txt ../test/src/test/resources/tpch/runtimeInfo/failedQueryIds.txt
+              cp -f test/src/test/resources/tpch/runtimeInfo/iterationTimes.txt ../test/src/test/resources/tpch/runtimeInfo/iterationTimes.txt
 
         - name: Show Old IGinX log
           if: always() && inputs.status != 'ok'
           shell: bash
-          run: cat IGinX/iginx-*.log
+          working-directory: IGinX
+          run: cat iginx-*.log
 
         - name: Stop Old IGinX
           if: inputs.status != 'ok'
@@ -70,6 +79,7 @@ runs:
         - name: Get Test Result
           id: get
           shell: bash
+          working-directory: IGinX
           run: |
               if [ -f test/src/test/resources/tpch/runtimeInfo/status.txt ]; then
                   STATUS=$(cat test/src/test/resources/tpch/runtimeInfo/status.txt)

.github/workflows/standard-test-suite.yml

@@ -11,72 +11,72 @@ concurrency:
     cancel-in-progress: true
 
 jobs:
-    unit-test:
-        uses: ./.github/workflows/unit-test.yml
-    unit-mds:
-        uses: ./.github/workflows/unit-mds.yml
-    case-regression:
-        uses: ./.github/workflows/case-regression.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-    standalone-test:
-        uses: ./.github/workflows/standalone-test.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-    standalone-test-no-optimizer:
-        uses: ./.github/workflows/standalone-test-no-optimizer.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-    db-ce:
-        uses: ./.github/workflows/DB-CE.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-    db-ce-no-optimizer:
-        uses: ./.github/workflows/DB-CE.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-            close-optimizer: "true"
-    standalone-test-vectordb:
-        uses: ./.github/workflows/standalone-test.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-            os-matrix: '["ubuntu-latest", "windows-latest"]'
-            db-matrix: '["VectorDB"]'
-            timeout-minutes: 300
-    standalone-test-no-optimizer-vectordb:
-        uses: ./.github/workflows/standalone-test-no-optimizer.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-            os-matrix: '["ubuntu-latest", "windows-latest"]'
-            db-matrix: '["VectorDB"]'
-            timeout-minutes: 300
-    db-ce-vectordb:
-        uses: ./.github/workflows/DB-CE.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-            os-matrix: '["ubuntu-latest", "windows-latest"]'
-            db-matrix: '["VectorDB"]'
-            functest: "NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT"
-            timeout-minutes: 360
-    db-ce-no-optimizer-vectordb:
-        uses: ./.github/workflows/DB-CE.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-            os-matrix: '["ubuntu-latest", "windows-latest"]'
-            db-matrix: '["VectorDB"]'
-            functest: "NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT"
-            timeout-minutes: 360
-            close-optimizer: "true"
-    remote-test:
-        uses: ./.github/workflows/remote-test.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
-    assembly-test:
-        uses: ./.github/workflows/assembly-test.yml
-    free-thread-test:
-        uses: ./.github/workflows/free-thread-test.yml
-        with:
-            metadata-matrix: '["zookeeper"]'
+#    unit-test:
+#        uses: ./.github/workflows/unit-test.yml
+#    unit-mds:
+#        uses: ./.github/workflows/unit-mds.yml
+#    case-regression:
+#        uses: ./.github/workflows/case-regression.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#    standalone-test:
+#        uses: ./.github/workflows/standalone-test.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#    standalone-test-no-optimizer:
+#        uses: ./.github/workflows/standalone-test-no-optimizer.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#    db-ce:
+#        uses: ./.github/workflows/DB-CE.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#    db-ce-no-optimizer:
+#        uses: ./.github/workflows/DB-CE.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#            close-optimizer: "true"
+#    standalone-test-vectordb:
+#        uses: ./.github/workflows/standalone-test.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#            os-matrix: '["ubuntu-latest", "windows-latest"]'
+#            db-matrix: '["VectorDB"]'
+#            timeout-minutes: 300
+#    standalone-test-no-optimizer-vectordb:
+#        uses: ./.github/workflows/standalone-test-no-optimizer.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#            os-matrix: '["ubuntu-latest", "windows-latest"]'
+#            db-matrix: '["VectorDB"]'
+#            timeout-minutes: 300
+#    db-ce-vectordb:
+#        uses: ./.github/workflows/DB-CE.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#            os-matrix: '["ubuntu-latest", "windows-latest"]'
+#            db-matrix: '["VectorDB"]'
+#            functest: "NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT"
+#            timeout-minutes: 360
+#    db-ce-no-optimizer-vectordb:
+#        uses: ./.github/workflows/DB-CE.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#            os-matrix: '["ubuntu-latest", "windows-latest"]'
+#            db-matrix: '["VectorDB"]'
+#            functest: "NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,CompactionIT,TimePrecisionIT,PySessionIT"
+#            timeout-minutes: 360
+#            close-optimizer: "true"
+#    remote-test:
+#        uses: ./.github/workflows/remote-test.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
+#    assembly-test:
+#        uses: ./.github/workflows/assembly-test.yml
+#    free-thread-test:
+#        uses: ./.github/workflows/free-thread-test.yml
+#        with:
+#            metadata-matrix: '["zookeeper"]'
     tpc-h-regression-test:
         uses: ./.github/workflows/tpc-h.yml
         with:
@@ -88,18 +88,18 @@ jobs:
             os-matrix: '["ubuntu-latest"]'
             metadata-matrix: '["zookeeper"]'
             close-optimizer: "true"
-    tpc-h-regression-test-vectordb:
-        uses: ./.github/workflows/tpc-h.yml
-        with:
-            os-matrix: '["ubuntu-latest"]'
-            metadata-matrix: '["zookeeper"]'
-            db-matrix: '["VectorDB"]'
-            timeout-minutes: 360
-    tpc-h-regression-test-no-optimizer-vectordb:
-        uses: ./.github/workflows/tpc-h.yml
-        with:
-            os-matrix: '["ubuntu-latest"]'
-            metadata-matrix: '["zookeeper"]'
-            db-matrix: '["VectorDB"]'
-            close-optimizer: "true"
-            timeout-minutes: 360
+#    tpc-h-regression-test-vectordb:
+#        uses: ./.github/workflows/tpc-h.yml
+#        with:
+#            os-matrix: '["ubuntu-latest"]'
+#            metadata-matrix: '["zookeeper"]'
+#            db-matrix: '["VectorDB"]'
+#            timeout-minutes: 360
+#    tpc-h-regression-test-no-optimizer-vectordb:
+#        uses: ./.github/workflows/tpc-h.yml
+#        with:
+#            os-matrix: '["ubuntu-latest"]'
+#            metadata-matrix: '["zookeeper"]'
+#            db-matrix: '["VectorDB"]'
+#            close-optimizer: "true"
+#            timeout-minutes: 360

arrow-patch/pom.xml

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    IGinX - the polystore system with high performance
+    Copyright (C) Tsinghua University
+    [email protected]
+
+    This program is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 3 of the License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with this program; if not, write to the Free Software Foundation,
+    Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>cn.edu.tsinghua</groupId>
+        <artifactId>iginx</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <artifactId>arrow-0patch</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>