chore(conf): change value of filesystem configuration data.config.write.buffer.timeout to 1 seconds

.github/actions/confWriter/action.yml

@@ -33,6 +33,10 @@ inputs:
         description: "the path of IGinX root directory"
         required: false
         default: "."
+    use-free-thread-python:
+        description: "use python3.13 free-threading mode"
+        required: false
+        default: "false"
     zookeeper-port:
         description: "zookeeper service port"
         required: false
@@ -67,12 +71,36 @@ runs:
               paths: ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-*/conf/config.properties
               statements: s/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g
 
+        - name: Find PythonCMD && Set Free-threading Support ENV
+          id: find_python_cmd
+          shell: bash -el {0}
+          run: |
+              if [[ "${{ inputs.use-free-thread-python }}" == "true" ]]; then
+                  if [ "$RUNNER_OS" == "Windows" ]; then
+                      pythonPath=$(echo "$FT_PYTHON_PATH" | sed 's/\\/\//g')
+                      pythonCMD="${pythonPath}/python3.13t.exe"
+                      echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV
+                  else
+                      pythonCMD='python3.13t'
+                      echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV
+                  fi
+              else
+                  if [ "$RUNNER_OS" == "Windows" ]; then
+                      pythonCMD=python
+                  else
+                      pythonCMD=python3
+                  fi
+              fi
+              # have to set env to make it work in ut tests
+              echo "pythonCMD=${pythonCMD}" >> $GITHUB_ENV
+              echo "pythonCMD=${pythonCMD}" >> $GITHUB_OUTPUT
+              echo ${pythonCMD}
+
         - name: Change Python CMD
-          if: runner.os == 'Windows'
           uses: ./.github/actions/edit
           with:
               paths: ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-*/conf/config.properties
-              statements: s/pythonCMD=python3/pythonCMD=python/g
+              statements: s|pythonCMD=python3|pythonCMD=${{ steps.find_python_cmd.outputs.pythonCMD }}|g
 
         - if: inputs.Close-Optimizer=='true'
           name: Close All Optimizer

.github/actions/context/action.yml

@@ -60,10 +60,27 @@ runs:
                 "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_outfile_macos.sh" "${{ inputs.DB-name }}" ${VERSION}
               fi
 
+        - if: inputs.shell=='client-after'
+          name: Install wget and 7zip
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Linux" ]; then
+                sudo apt-get install -y p7zip-full
+                sudo apt-get install -y wget
+              elif [ "$RUNNER_OS" == "Windows" ]; then
+                choco install 7zip -y
+                choco install wget -y
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                brew install p7zip
+                brew install wget
+              fi
+
         - if: inputs.shell=='client-after'
           name: Pre Test Client Import File
           shell: bash
           run: |
+              wget https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/bigcsv.7z
+              7z x bigcsv.7z
               if [ "$RUNNER_OS" == "Linux" ]; then
                 chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_infile.sh"
                 "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_infile.sh" ${VERSION}

.github/actions/dbRunner/action.yml

@@ -63,6 +63,8 @@ runs:
           name: Run DB
           shell: bash
           run: |
+              java -version
+              which java
               if [ "$RUNNER_OS" == "Linux" ]; then
                 chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12.sh"
                 "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12.sh" ${{ inputs.IoTDB12-ports }}

.github/actions/dependence/action.yml

@@ -4,7 +4,7 @@ inputs:
     python-version:
         description: "python version"
         required: false
-        default: 3.9
+        default: 3.11
     java:
         description: "java version"
         required: false
@@ -22,6 +22,17 @@ inputs:
         description: "is docker needed in this test"
         required: false
         default: "false"
+    free-thread-python-required:
+        description: "is free-threading python needed"
+        required: false
+        default: "false"
+    iginx-conda-flag:
+        description: "whether to use conda"
+        required: false
+        default: "false"
+    iginx-conda-env:
+        description: "conda env name"
+        required: false
 
 runs:
     using: "composite"
@@ -34,6 +45,19 @@ runs:
               tzutil /s "China Standard Time"
               echo "JAVA_OPTS=-Xmx4g -Xms2g" >> %GITHUB_ENV%
 
+        # after this setup, every shell using login mode will be in conda env
+        # conda should be set at very beginning because this action will delete ~/.bashrc & ~/.bash_profile
+        # and regenerate them by initializing conda
+        - if: inputs.scope=='all' && inputs.iginx-conda-flag == 'true'
+          name: Setup Miniconda
+          uses: conda-incubator/[email protected]
+          with:
+              miniconda-version: ${{ inputs.conda-version }}
+              python-version: ${{ inputs.python-version }}
+              activate-environment: ${{ inputs.iginx-conda-env }}
+              channel-priority: strict
+              run-post: "false"
+
         # we use a patched version of lima(slower) because colima cannot start on M1 chips and the docker task is lightweight.
         - if: runner.os == 'macOS' &&  inputs.docker-required=='true'
           name: Install Docker on MacOS
@@ -60,17 +84,38 @@ runs:
               echo "requests" >> requirements.txt
               echo "torch" >> requirements.txt
 
-        - if: inputs.scope=='all'
+        - if: inputs.scope=='all' && inputs.iginx-conda-flag != 'true'
           name: Set up Python ${{ inputs.python-version }}
           uses: actions/setup-python@v5
           with:
               python-version: ${{ inputs.python-version }}
               cache: "pip" # caching pip dependencies
 
+        # when using login mode shell, setup-python does not take effect.
+        # thus manually add python path to PATH
+        - if: inputs.scope=='all' && inputs.iginx-conda-flag != 'true'
+          name: Add System Python to PATH
+          shell: bash
+          run: |
+              PYTHON_PATH=$(dirname $(which python))
+              echo $PYTHON_PATH
+              echo "IGINX_PYTHON_PATH=$PYTHON_PATH" >> $GITHUB_ENV
+
         - if: inputs.scope=='all'
           name: Install Python dependencies
-          shell: bash
-          run: pip install -r requirements.txt
+          shell: bash -el {0}
+          run: |
+              if [ ! -z "$IGINX_PYTHON_PATH" ]; then
+                export PATH="$IGINX_PYTHON_PATH:$PATH"
+              fi
+              echo $PATH
+              which python
+              python -m pip install --upgrade pip
+              pip install -r requirements.txt
+
+        - if: inputs.free-thread-python-required=='true'
+          name: Install free-thread python
+          uses: ./.github/actions/service/freeThreadPython
 
         - name: Set up JDK ${{ inputs.java }}
           uses: actions/setup-java@v4
@@ -79,6 +124,26 @@ runs:
               distribution: ${{ runner.os == 'macOS' && matrix.java == '8' && 'liberica' || 'temurin' }}
               cache: "maven"
 
+        - name: Check Java & Python Version
+          shell: bash
+          run: |
+              java -version
+              which java
+              python --version
+              which python
+
+        - name: Check Java & Python Version in Conda and set conda env name
+          if: inputs.iginx-conda-flag == 'true'
+          shell: bash -el {0}
+          run: |
+              java -version
+              which java
+              python --version
+              which python
+              # to access conda env in scripts
+              echo "IGINX_CONDA_ENV=${{ inputs.iginx-conda-env }}" >> $GITHUB_ENV
+              echo "IGINX_CONDA_FLAG=${{ inputs.iginx-conda-flag }}" >> $GITHUB_ENV
+
         - name: Get project info
           id: project
           uses: ./.github/actions/project

.github/actions/iginxRunner/action.yml

@@ -16,7 +16,7 @@ runs:
         # TODO: use iginx binaries in assembly rather than core
         - if: inputs.if-test-udf=='true'
           name: Test UDF Path
-          shell: bash
+          shell: bash -el {0}
           run: |
               if [ "$RUNNER_OS" == "Linux" ]; then
                 sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties
@@ -34,8 +34,9 @@ runs:
 
         - if: inputs.if-test-udf=='false' && inputs.if-stop=='false'
           name: Start IGinX
-          shell: bash
+          shell: bash -el {0}
           run: |
+              which python
               if [ "$RUNNER_OS" == "Linux" ]; then
                 chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh"
                 "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888

.github/actions/service/freeThreadPython/action.yml

@@ -0,0 +1,115 @@
+name: "Install_Free-thread_Python"
+description: "Install Free-thread Python"
+
+runs:
+    using: "composite"
+    steps:
+        - name: Set Python variables
+          id: set-python-vars
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Windows" ]; then
+                echo "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.13.0/python-3.13.0-amd64.exe" >> $GITHUB_ENV
+                echo "PYTHON_INSTALLER_PATH=python-installer.exe" >> $GITHUB_ENV
+                echo "CACHE_KEY=python-windows-3.13.0" >> $GITHUB_ENV
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                echo "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.13.0/python-3.13.0-macos11.pkg" >> $GITHUB_ENV
+                echo "PYTHON_INSTALLER_PATH=python-installer.pkg" >> $GITHUB_ENV
+                echo "CACHE_KEY=python-macos-3.13.0" >> $GITHUB_ENV
+              fi
+
+        - name: Restore Python installer
+          if: runner.os!='Linux'
+          id: restore-cache-python
+          uses: actions/cache/restore@v3
+          with:
+              path: ${{ env.PYTHON_INSTALLER_PATH }}
+              key: ${{ env.CACHE_KEY }}
+
+        - name: Download Python installer
+          if: steps.restore-cache-python.outputs.cache-hit != 'true' && runner.os=='Windows'
+          shell: powershell
+          run: |
+              Invoke-WebRequest -Uri $env:PYTHON_INSTALLER_URL -OutFile $env:PYTHON_INSTALLER_PATH
+
+        # have to wait until installation finishes
+        - if: runner.os=='Windows'
+          name: Setup Python3.13 on Windows
+          shell: powershell
+          run: |
+              Start-Process -FilePath "$(Join-Path -Path $PWD -ChildPath $env:PYTHON_INSTALLER_PATH)" -ArgumentList "/quiet","Include_freethreaded=1" -Wait
+              $pythonPath = "$env:LocalAppData\Programs\Python\Python313"
+              Get-ChildItem $pythonPath
+              $env:Path += ";$pythonPath"
+              echo "$pythonPath" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+              echo "FT_PYTHON_PATH=$pythonPath" | Out-File -FilePath $env:GITHUB_ENV -Append
+              python3.13t -VV
+
+        - if: runner.os=='Linux'
+          name: Setup Python3.13 on Linux
+          shell: bash
+          run: |
+              sudo add-apt-repository ppa:deadsnakes
+              sudo apt-get update
+              sudo apt-get install python3.13-nogil python3.13-dev
+              python3.13t -VV
+
+        - name: Download Python installer
+          if: steps.restore-cache-python.outputs.cache-hit != 'true' && runner.os=='macOS'
+          shell: bash
+          run: |
+              curl -o $PYTHON_INSTALLER_PATH $PYTHON_INSTALLER_URL
+
+        - if: runner.os=='macOS'
+          name: Setup Python3.13 on MacOS
+          shell: bash
+          run: |
+              cat > ./choicechanges.plist <<EOF
+              <?xml version="1.0" encoding="UTF-8"?>
+              <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+              <plist version="1.0">
+              <array>
+                  <dict>
+                      <key>attributeSetting</key>
+                      <integer>1</integer>
+                      <key>choiceAttribute</key>
+                      <string>selected</string>
+                      <key>choiceIdentifier</key>
+                      <string>org.python.Python.PythonTFramework-3.13</string>
+                  </dict>
+              </array>
+              </plist>
+              EOF
+              sudo installer -pkg ./$PYTHON_INSTALLER_PATH \
+                  -applyChoiceChangesXML ./choicechanges.plist \
+                  -target /
+              python3.13t -VV
+              export PATH="$PATH:/Library/Frameworks/PythonT.framework/Versions/3.13/bin"
+
+        - name: Cache Python installer
+          if: runner.os!='Linux' && steps.restore-cache-python.outputs.cache-hit != 'true'
+          id: cache-python
+          uses: actions/cache/save@v3
+          with:
+              path: ${{ env.PYTHON_INSTALLER_PATH }}
+              key: ${{ env.CACHE_KEY }}
+
+        - name: Install pip
+          if: runner.os=='Linux'
+          shell: bash
+          run: |
+              curl -O https://bootstrap.pypa.io/get-pip.py
+              sudo python3.13t get-pip.py
+
+        # pandas 2.2.3 cannot be directly installed on windows yet. use dev wheel file
+        - name: Install dependencies for tests
+          shell: bash
+          run: |
+              curl -L -O https://github.com/IGinX-THU/IGinX-resources/raw/refs/heads/main/resources/python/pandas-3.0.0.dev0+1654.g32a97a969a-cp313-cp313t-win_amd64.whl
+              ls -l
+              if [ "$RUNNER_OS" == "Windows" ]; then
+               python3.13t -m pip install numpy thrift pemjax
+               python3.13t -m pip install pandas*.whl
+              else
+               sudo python3.13t -m pip install pandas==2.2.3 numpy thrift pemjax
+              fi

.github/actions/tpchSingleTest/action.yml

@@ -18,12 +18,12 @@ runs:
 
         - name: Run SQL Warmup on New IGinX
           if: inputs.status != 'ok'
-          shell: bash
+          shell: bash -el {0}
           run: mvn test -q -Dtest=SQLWarmupIT -DfailIfNoTests=false -P-format
 
         - name: Run TPCH Test on New IGinX
           if: inputs.status != 'ok'
-          shell: bash
+          shell: bash -el {0}
           run: mvn test -q -Dtest=TPCHNewIT -DfailIfNoTests=false -P-format
 
         - name: Show New IGinX log
@@ -39,7 +39,7 @@ runs:
 
         - name: Start Old IGinX
           if: inputs.status != 'ok'
-          shell: bash
+          shell: bash -el {0}
           working-directory: IGinX
           run: |
               chmod +x core/target/iginx-core-*/sbin/start_iginx.sh
@@ -48,12 +48,12 @@ runs:
 
         - name: Run SQL Warmup on Old IGinX
           if: inputs.status != 'ok'
-          shell: bash
+          shell: bash -el {0}
           run: mvn test -q -Dtest=SQLWarmupIT -DfailIfNoTests=false -P-format
 
         - name: Run TPCH Test on Old IGinX
           if: inputs.status != 'ok'
-          shell: bash
+          shell: bash -el {0}
           run: mvn test -q -Dtest=TPCHOldIT -DfailIfNoTests=false -P-format
 
         - name: Show Old IGinX log

.github/scripts/dataSources/restart/influxdb.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # IGinX - the polystore system with high performance
 # Copyright (C) Tsinghua University
@@ -22,6 +22,11 @@
 
 port=$1
 echo "Starting InfluxDB on port $port"
-sudo sh -c "cd influxdb2-2.0.7-linux-amd64-$port/; nohup ./influxd run --bolt-path=~/.influxdbv2/influxd.bolt --engine-path=~/.influxdbv2/engine --http-bind-address=:$port --query-memory-bytes=20971520 --query-concurrency=2 > influxdb_$port.log 2>&1 &"
+sudo sh -c "cd influxdb2-2.0.7-linux-amd64-$port/; nohup ./influxd run --bolt-path=~/.influxdbv2/influxd.bolt --engine-path=~/.influxdbv2/engine --http-bind-address=:$port --query-memory-bytes=300971520 --query-concurrency=2 > influxdb_$port.log 2>&1 &"
 sleep 10
 
+
+sudo lsof -i:$port
+if [ $? -eq 1 ]; then
+    echo "No process is listening on port $port"
+fi