merge main

aqni · aqni · commit 5b150a86e034 · 2025-03-06T16:00:10.000+08:00
diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml
@@ -79,23 +79,16 @@ runs:
           id: find_python_cmd
           shell: bash -el {0}
           run: |
-              if [[ "${{ inputs.use-free-thread-python }}" == "true" ]]; then
-                  if [ "$RUNNER_OS" == "Windows" ]; then
-                      pythonPath=$(echo "$FT_PYTHON_PATH" | sed 's/\\/\//g')
-                      pythonCMD="${pythonPath}/python3.13t.exe"
-                      echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV
-                  else
-                      pythonCMD='python3.13t'
-                      echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV
-                  fi
+              if [ "${{ inputs.use-free-thread-python }}" == "true" ]; then
+                  echo "TORCH_SUPPORTED=false" >> $GITHUB_ENV
+              fi
+              if [ "$RUNNER_OS" == "Windows" ]; then
+                  pythonCMD=python
               else
-                  if [ "$RUNNER_OS" == "Windows" ]; then
-                      pythonCMD=python
-                  else
-                      pythonCMD=python3
-                  fi
+                  pythonCMD=python3
               fi
               # have to set env to make it work in ut tests
+              ${pythonCMD} -VV
               echo "pythonCMD=${pythonCMD}" >> $GITHUB_ENV
               echo "pythonCMD=${pythonCMD}" >> $GITHUB_OUTPUT
               echo ${pythonCMD}
diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml
@@ -116,7 +116,9 @@ runs:
 
         - if: inputs.free-thread-python-required=='true'
           name: Install free-thread python
-          uses: ./.github/actions/service/freeThreadPython
+          uses: ./.github/actions/service/freeThreadConda
+          with:
+              iginx-conda-env: ${{ inputs.iginx-conda-env }}
 
         - name: Set up JDK ${{ inputs.java }}
           uses: actions/setup-java@v4
diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml
@@ -9,6 +9,17 @@ inputs:
         description: "to test UDF path detection"
         required: false
         default: "false"
+    iginx-conda-flag:
+        # this step is only needed when using python3.13 by conda because latest conda supports py3.12,
+        # and we use conda-forge to support 3.13. Thus, the required env would not be activated automatically
+        # even when we use login mode of bash
+        description: "whether to manually activate conda env before start IGinX"
+        required: false
+        default: "false"
+    iginx-conda-env:
+        description: "required conda env name"
+        required: false
+        default: "false"
 
 runs:
     using: "composite" # Mandatory parameter
@@ -36,16 +47,18 @@ runs:
           name: Start IGinX
           shell: bash -el {0}
           run: |
-              which python
+              if [ "${{ inputs.iginx-conda-flag }}" == "true" ]; then
+                  conda activate ${{ inputs.iginx-conda-env }}
+              fi
               if [ "$RUNNER_OS" == "Linux" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888
+                  chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh"
+                  "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 7888
               elif [ "$RUNNER_OS" == "Windows" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888
+                  chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh"
+                  "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 7888
               elif [ "$RUNNER_OS" == "macOS" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888
+                  chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh"
+                  "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 7888
               fi
 
         - if: inputs.if-test-udf=='false' && inputs.if-stop=='true'
diff --git a/.github/actions/service/freeThreadConda/action.yml b/.github/actions/service/freeThreadConda/action.yml
@@ -0,0 +1,38 @@
+name: "Install_Free-thread_Python in Conda"
+description: "Install Free-thread Python in Conda"
+inputs:
+    iginx-conda-env:
+        description: "conda env name"
+        required: false
+
+runs:
+    using: "composite"
+    steps:
+        # this step creates env test(default python version) which will not be used
+        - uses: conda-incubator/setup-miniconda@v3
+          with:
+              channels: conda-forge
+              channel-priority: flexible
+              conda-remove-defaults: "true"
+              run-post: "false"
+
+        # currently we can only install free-thread python 3.13 by conda-forge. latest miniconda only supports 3.12
+        # thus in action steps ahead, remember to 'conda activate $ENV_NAME' at start if conda should be used
+        - name: Check channels & install python3.13
+          shell: bash -el {0}
+          run: |
+              conda config --show channels
+              conda create -n ${{ inputs.iginx-conda-env }} python=3.13 python-freethreading -c conda-forge -q
+              conda activate ${{ inputs.iginx-conda-env }}
+              python -VV
+              python -c "import sys;print(sys._is_gil_enabled())"
+              curl -L -O https://github.com/IGinX-THU/IGinX-resources/raw/refs/heads/main/resources/python/pandas-3.0.0.dev0+1654.g32a97a969a-cp313-cp313t-win_amd64.whl
+              curl -L -O https://pypi.fury.io/arrow-nightlies/-/ver_1AsSYu/pyarrow-19.0.0.dev286-cp313-cp313t-win_amd64.whl#md5=3c940f7b4ffd0debaa8b23d06bc9b2ea
+              ls -l
+              if [ "$RUNNER_OS" == "Windows" ]; then
+               python -m pip install numpy thrift pemjax
+               python -m pip install pandas*.whl
+               python3.13t -m pip install pyarrow*.whl
+              else
+               python -m pip install pandas==2.2.3 numpy thrift pemjax
+              fi
diff --git a/.github/actions/service/freeThreadPython/action.yml b/.github/actions/service/freeThreadPython/action.yml
diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh
@@ -28,7 +28,8 @@ sed -i "s/restPort=[0-9]\+/restPort=$2/g" core/target/iginx-core-*/conf/config.p
 
 sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh"
 
-sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &"
+python3 -VV
+bash -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &"
 
 sh -c "sleep 3"
 
diff --git a/.github/scripts/iginx/iginx_macos.sh b/.github/scripts/iginx/iginx_macos.sh
@@ -32,7 +32,8 @@ echo "JAVA_HOME is set to $JAVA_HOME"
 
 sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh"
 
-sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &"
+python3 -VV
+bash -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &"
 
 sh -c "sleep 3"
 
diff --git a/.github/scripts/test/test_union.sh b/.github/scripts/test/test_union.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
  
-
+python -VV
 pwd
 
 cd ..
diff --git a/.github/workflows/free-thread-test.yml b/.github/workflows/free-thread-test.yml
@@ -35,6 +35,10 @@ jobs:
                 metadata: ${{ fromJSON(inputs.metadata-matrix) }}
                 DB-name: ${{ fromJSON(inputs.db-matrix) }}
         runs-on: ${{ matrix.os }}
+        env:
+            # only used in this workflow file
+            IGINX_CONDA_FLAG: "true"
+            IGINX_CONDA_ENV: "iginxEnv"
         steps:
             - uses: actions/checkout@v4
               # use free-threading python instead of stable version
@@ -44,6 +48,8 @@ jobs:
                   java: ${{ matrix.java }}
                   free-thread-python-required: "true"
                   scope: "only-java"
+                  iginx-conda-flag: ${{ env.IGINX_CONDA_FLAG }}
+                  iginx-conda-env: ${{ env.IGINX_CONDA_ENV }}
 
             - name: Run Metadata
               uses: ./.github/actions/metadataRunner
@@ -70,13 +76,17 @@ jobs:
 
             - name: Start IGinX
               uses: ./.github/actions/iginxRunner
+              with:
+                  iginx-conda-flag: ${{ env.IGINX_CONDA_FLAG }}
+                  iginx-conda-env: ${{ env.IGINX_CONDA_ENV }}
 
             - name: TestController IT
               if: always()
-              shell: bash
+              shell: bash -el {0}
               env:
                   METADATA_STORAGE: ${{ matrix.metadata }}
               run: |
+                  conda activate ${{ env.IGINX_CONDA_ENV }}
                   chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh"
                   mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format
 
diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/etcd/ETCDMetaStorage.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/etcd/ETCDMetaStorage.java
@@ -2364,7 +2364,7 @@ public void updateJobTrigger(TriggerDescriptor descriptor) throws MetaStorageExc
       throw new MetaStorageException(e);
     } finally {
       if (jobTriggerLease != -1) {
-        releaseTransform();
+        releaseJobTrigger();
       }
     }
     if (jobTriggerChangeHook != null) {
diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java
@@ -250,13 +250,20 @@ private void verifyJobState(long jobId, JobState expectedState) throws SessionEx
   private void verifyJobFinishedBlocked(long jobId) throws SessionException, InterruptedException {
     LOGGER.info("job is {}", jobId);
     JobState jobState = JobState.JOB_CREATED;
+    int timeout = 60000; // 60s
     while (!jobState.equals(JobState.JOB_CLOSED)
         && !jobState.equals(JobState.JOB_FAILED)
-        && !jobState.equals(JobState.JOB_FINISHED)) {
+        && !jobState.equals(JobState.JOB_FINISHED)
+        && timeout > 0) {
       Thread.sleep(500);
+      timeout -= 500;
       jobState = session.queryTransformJobStatus(jobId);
     }
     LOGGER.info("job {} state is {}", jobId, jobState.toString());
+    if (timeout <= 0) {
+      LOGGER.error("job is stuck, check IGinX server log...");
+      fail();
+    }
 
     if (!needCompareResult) {
       return;
diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java
diff --git a/test/src/test/resources/udf/no_mod_udf.py b/test/src/test/resources/udf/no_mod_udf.py

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@`
`17`	`17`	`# along with this program. If not, see <http://www.gnu.org/licenses/>.`
`18`	`18`	`#`
`19`	`19`
`20`		`-`
	`20`	`+python -VV`
`21`	`21`	`pwd`
`22`	`22`
`23`	`23`	`cd ..`
Original file line number	Diff line number	Diff line change
`@@ -2364,7 +2364,7 @@ public void updateJobTrigger(TriggerDescriptor descriptor) throws MetaStorageExc`
`2364`	`2364`	`throw new MetaStorageException(e);`
`2365`	`2365`	`} finally {`
`2366`	`2366`	`if (jobTriggerLease != -1) {`
`2367`		`- releaseTransform();`
	`2367`	`+ releaseJobTrigger();`
`2368`	`2368`	`}`
`2369`	`2369`	`}`
`2370`	`2370`	`if (jobTriggerChangeHook != null) {`