Ci/check update pysesssion

.github/actions/confWriter/action.yml

@@ -17,6 +17,10 @@ inputs:
         description: "make the storage engine read-only"
         required: false
         default: "false"
+    Has-Data:
+        description: "does the storage engine has data"
+        required: false
+        default: "false"
     Push-Down:
         description: "make the IGinX push down filter"
         required: false
@@ -29,18 +33,22 @@ inputs:
         description: "the path of IGinX root directory"
         required: false
         default: "${GITHUB_WORKSPACE}"
+    zookeeper-port:
+        description: "zookeeper service port"
+        required: false
+        default: "2181"
 
 runs:
     using: "composite" # Mandatory parameter
     steps:
-        - if: inputs.DB-name=='FileStore'
-          name: save config for FileStore
+        - if: inputs.DB-name=='FileSystem'
+          name: save config for FileSystem
           shell: bash
           run: |
               cp -f "${{ inputs.Root-Dir-Path }}/conf/config.properties" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties"
 
-        - if: inputs.DB-name=='FileStore'
-          name: save config for FileStore
+        - if: inputs.DB-name=='FileSystem'
+          name: save config for FileSystem
           shell: bash
           run: |
               cp -f "${{ inputs.Root-Dir-Path }}/conf/config.properties" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties"
@@ -74,6 +82,21 @@ runs:
           run: |
               echo "${{ inputs.DB-name }}" > ${{ inputs.Root-Dir-Path }}/test/src/test/resources/DBName.txt
 
+        - if: inputs.zookeeper-port!='2181'
+          name: Change Zookeeper Port
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Linux" ]; then
+                sudo sed -i 's/zookeeperConnectionString=127.0.0.1:2181/zookeeperConnectionString=127.0.0.1:${{ inputs.zookeeper-port }}/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              elif [ "$RUNNER_OS" == "Windows" ]; then
+                sed -i 's/zookeeperConnectionString=127.0.0.1:2181/zookeeperConnectionString=127.0.0.1:${{ inputs.zookeeper-port }}/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                sudo sed -i '' 's/zookeeperConnectionString=127.0.0.1:2181/zookeeperConnectionString=127.0.0.1:${{ inputs.zookeeper-port }}/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              else
+                echo "$RUNNER_OS is not supported"
+                exit 1
+              fi
+
         - name: Change UDF conf
           shell: bash
           run: |
@@ -103,25 +126,35 @@ runs:
                 echo "$RUNNER_OS is not supported"
                 exit 1
               fi
+
+        - if: inputs.Has-Data=='true'
+          name: Set Has-Data
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Linux" ]; then
+                sudo sed -i 's/has_data=false/has_data=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              elif [ "$RUNNER_OS" == "Windows" ]; then
+                sed -i 's/has_data=false/has_data=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                sudo sed -i '' 's/has_data=false/has_data=true/' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+              else
+                echo "$RUNNER_OS is not supported"
+                exit 1
+              fi
+
         - if: inputs.Push-Down=='true'
           name: Change push_down
           shell: bash
           run: |
               if [ "$RUNNER_OS" == "Linux" ]; then
                 sudo sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i 's/FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off/FilterPushDownAddSchemaPrefixRule=on,FilterPushDownAddSchemaPrefixRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i 's/FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off/FilterPushDownPathUnionJoinRule=on,FilterPushDownProjectReorderSortRule=on,FilterPushDownRenameRule=on,FilterPushDownSelectRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i 's/FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off/FilterPushDownSetOpRule=on,FilterPushDownTransformRule=on,FilterPushIntoJoinConditionRule=on,FilterPushOutJoinConditionRule=on,FilterPushDownGroupByRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sed -i 's/FilterPushDownRule=off/FilterPushDownRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               elif [ "$RUNNER_OS" == "Windows" ]; then
                 sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i 's/FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off/FilterPushDownAddSchemaPrefixRule=on,FilterPushDownAddSchemaPrefixRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i 's/FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off/FilterPushDownPathUnionJoinRule=on,FilterPushDownProjectReorderSortRule=on,FilterPushDownRenameRule=on,FilterPushDownSelectRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i 's/FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off/FilterPushDownSetOpRule=on,FilterPushDownTransformRule=on,FilterPushIntoJoinConditionRule=on,FilterPushOutJoinConditionRule=on,FilterPushDownGroupByRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sed -i 's/FilterPushDownRule=off/FilterPushDownRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               elif [ "$RUNNER_OS" == "macOS" ]; then
                 sudo sed -i '' 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i '' 's/FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off/FilterPushDownAddSchemaPrefixRule=on,FilterPushDownAddSchemaPrefixRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i '' 's/FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off/FilterPushDownPathUnionJoinRule=on,FilterPushDownProjectReorderSortRule=on,FilterPushDownRenameRule=on,FilterPushDownSelectRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i '' 's/FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off/FilterPushDownSetOpRule=on,FilterPushDownTransformRule=on,FilterPushIntoJoinConditionRule=on,FilterPushOutJoinConditionRule=on,FilterPushDownGroupByRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sed -i '' 's/FilterPushDownRule=off/FilterPushDownRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               else
                 echo "$RUNNER_OS is not supported"
                 exit 1

.github/actions/dbConfWriter/action.yml

@@ -5,6 +5,10 @@ inputs:
         description: "DB name"
         required: false
         default: IoTDB12
+    DB-port:
+        description: "DB port"
+        required: false
+        default: "0"
     Root-Dir-Path:
         description: "the path of IGinX root directory"
         required: false
@@ -47,13 +51,13 @@ runs:
               paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
               statements: s/#storageEngineList=127.0.0.1#8086/storageEngineList=127.0.0.1#8086/g
 
-        - if: inputs.DB-name == 'FileStore'
+        - if: inputs.DB-name == 'FileSystem'
           name: Modify IGinX Config
           uses: ./.github/actions/edit
           with:
               paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
               statements: |
-                  s/^#storageEngineList=127.0.0.1#6667#filestore/storageEngineList=127.0.0.1#6667#filestore/g
+                  s/^#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#6667#filesystem/g
                   s#dir=data#dir=${PWD}/test/iginx_mn#g
                   s/data.config.write.buffer.size=104857600/data.config.write.buffer.size=1048576/g
                   s/write.buffer.size=104857600/write.buffer.size=1048576/g
@@ -97,3 +101,23 @@ runs:
           with:
               paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
               statements: s|^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path|storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=${{ steps.mysql-properties.outputs.path }}|g
+
+        # use regex
+        - if: inputs.DB-port!='0'
+          name: Change db port
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Linux" ]; then
+                sudo sed -i -E "s/^(storageEngineList=127\.0\.0\.1#)[0-9]+(#.*)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+                sudo sed -i -E "s/(mongodb:\/\/127\.0\.0\.1:)[0-9]+(\/)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              elif [ "$RUNNER_OS" == "Windows" ]; then
+                sed -i -E "s/^(storageEngineList=127\.0\.0\.1#)[0-9]+(#.*)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+                sed -i -E "s/(mongodb:\/\/127\.0\.0\.1:)[0-9]+(\/)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                sudo sed -i '' -E "s/^(storageEngineList=127\.0\.0\.1#)[0-9]+(#.*)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+                sudo sed -i '' -E "s/(mongodb:\/\/127\.0\.0\.1:)[0-9]+(\/)/\1${{ inputs.DB-port }}\2/" ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              else
+                echo "$RUNNER_OS is not supported"
+                exit 1
+              fi
+              cat ${{ inputs.Root-Dir-Path }}/conf/config.properties

.github/actions/dbRunner/action.yml

@@ -169,19 +169,19 @@ runs:
               paths: conf/config.properties
               statements: s|^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path|storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=${{ steps.mysql-properties.outputs.path }}|g
 
-        - if: inputs.DB-name=='FileStore'
+        - if: inputs.DB-name=='FileSystem'
           name: Run DB
           shell: bash
           run: |
               cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/conf/config.properties.bak"
 
-        - if: inputs.DB-name == 'FileStore'
+        - if: inputs.DB-name == 'FileSystem'
           name: Modify IGinX Config
           uses: ./.github/actions/edit
           with:
               paths: conf/config.properties
               statements: |
-                  s/^#storageEngineList=127.0.0.1#6667#filestore/storageEngineList=127.0.0.1#6667#filestore/g
+                  s/^#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#6667#filesystem/g
                   s#dir=data#dir=${{ steps.project.outputs.workspace }}/test/iginx_mn#g
                   s#dummy_dir=dummy#dummy_dir=${{ steps.project.outputs.workspace }}/test/mn#g
                   s/data.config.write.buffer.size=104857600/data.config.write.buffer.size=1048576/g

.github/actions/dependence/action.yml

@@ -40,7 +40,7 @@ runs:
           shell: bash
           run: |
               brew update
-              brew install docker
+              brew install docker qemu
               brew install colima
               LIMACTL_PATH=$(brew --prefix)/bin/limactl
               sudo curl -L -o $LIMACTL_PATH https://github.com/mikekazakov/lima-nohvf/raw/master/limactl && sudo chmod +x $LIMACTL_PATH

.github/actions/service/mysql/action.yml

@@ -40,5 +40,5 @@ runs:
           run: |
               for port in ${{ inputs.ports }}; do
                 mysqld --defaults-file=./$port.ini --initialize-insecure
-                mysqld --defaults-file=./$port.ini &
+                mysqld --defaults-file=./$port.ini & echo $! > ./mysql_$port.pid
               done

.github/actions/service/portMapper/action.yml

@@ -0,0 +1,66 @@
+name: "Database Ports Mapper"
+description: "Map database name to corresponding ports"
+inputs:
+    DB-name:
+        description: "Name of the database to get ports for"
+        required: true
+
+outputs:
+    port1:
+        description: "port1"
+        value: ${{ steps.get-ports.outputs.port1 }}
+    port2:
+        description: "port2"
+        value: ${{ steps.get-ports.outputs.port2 }}
+    port3:
+        description: "port3"
+        value: ${{ steps.get-ports.outputs.port3 }}
+
+runs:
+    using: "composite"
+    steps:
+        - id: get-ports
+          shell: bash
+          run: |
+              case "${{ inputs.DB-name }}" in
+                "InfluxDB")
+                  echo "port1=8086" >> $GITHUB_OUTPUT
+                  echo "port2=8087" >> $GITHUB_OUTPUT
+                  echo "port3=8088" >> $GITHUB_OUTPUT
+                  ;;
+                "IoTDB12")
+                  echo "port1=6667" >> $GITHUB_OUTPUT
+                  echo "port2=6668" >> $GITHUB_OUTPUT
+                  echo "port3=6669" >> $GITHUB_OUTPUT
+                  ;;
+                "FileSystem")
+                  echo "port1=6667" >> $GITHUB_OUTPUT
+                  echo "port2=6668" >> $GITHUB_OUTPUT
+                  echo "port3=6669" >> $GITHUB_OUTPUT
+                  ;;
+                "MySQL")
+                  echo "port1=3306" >> $GITHUB_OUTPUT
+                  echo "port2=3307" >> $GITHUB_OUTPUT
+                  echo "port3=3308" >> $GITHUB_OUTPUT
+                  ;;
+                "PostgreSQL")
+                  echo "port1=5432" >> $GITHUB_OUTPUT
+                  echo "port2=5433" >> $GITHUB_OUTPUT
+                  echo "port3=5434" >> $GITHUB_OUTPUT
+                  ;;
+                "MongoDB")
+                  echo "port1=27017" >> $GITHUB_OUTPUT
+                  echo "port2=27018" >> $GITHUB_OUTPUT
+                  echo "port3=27019" >> $GITHUB_OUTPUT
+                  ;;
+                "Redis")
+                  echo "port1=6379" >> $GITHUB_OUTPUT
+                  echo "port2=6380" >> $GITHUB_OUTPUT
+                  echo "port3=6381" >> $GITHUB_OUTPUT
+                  ;;
+                *)
+                  echo "port1=0" >> $GITHUB_OUTPUT
+                  echo "port2=0" >> $GITHUB_OUTPUT
+                  echo "port3=0" >> $GITHUB_OUTPUT
+                  ;;
+              esac

.github/actions/service/zookeeper/action.yml

@@ -10,30 +10,35 @@ inputs:
     start:
         description: "whether to start"
         required: true
+    port:
+        description: "zk port"
+        required: false
+        default: "2181"
 
 runs:
     using: "composite"
     steps:
         - if: fromJSON(inputs.stop)
           name: Stop zookeeper
           shell: bash
-          working-directory: ${{ github.action_path }}
-          run: zkServer.sh --config conf stop
+          working-directory: ${{ runner.temp }}/zookeeper-${{ inputs.port }}
+          run: bin/zkServer.sh --config conf stop
 
         - if: fromJSON(inputs.clean)
           name: Clean zookeeper
           shell: bash
-          working-directory: ${{ github.action_path }}
+          working-directory: ${{ runner.temp }}/zookeeper-${{ inputs.port }}
           run: rm -rf data logs conf
 
         - if: fromJSON(inputs.start)
           name: Start zookeeper
           shell: bash
-          working-directory: ${{ github.action_path }}
+          working-directory: ${{ runner.temp }}/zookeeper-${{ inputs.port }}
           run: |
               mkdir -p conf
               echo "dataDir=data" > conf/zoo.cfg
               echo "dataLogDir=logs" >> conf/zoo.cfg
-              echo "clientPort=2181" >> conf/zoo.cfg
+              echo "clientPort=${{ inputs.port }}" >> conf/zoo.cfg
+              echo "admin.serverPort=$((${{ inputs.port }} + 10000))" >> conf/zoo.cfg
 
-              zkServer.sh --config conf start
+              bin/zkServer.sh --config conf start

.github/actions/setup/zookeeper/action.yml

@@ -5,6 +5,10 @@ inputs:
         description: "zookeeper version"
         required: false
         default: "3.7.2"
+    port:
+        description: "zk port"
+        required: false
+        default: "2181"
 
 runs:
     using: "composite"
@@ -32,10 +36,12 @@ runs:
               subdir: "apache-zookeeper-{version}-bin"
               ext: "tar.gz"
 
-        - name: Add zookeeper Bin to PATH
+        - name: Copy ZK to Port Dir
           shell: bash
-          working-directory: ${{ steps.base.outputs.tool-path }}
-          run: echo "$PWD/bin" >> $GITHUB_PATH
+          run: |
+              if [ ! -d "${{ runner.temp }}/zookeeper-${{ inputs.port }}" ]; then
+                cp -r "${{ steps.base.outputs.tool-path }}" "${{ runner.temp }}/zookeeper-${{ inputs.port }}"
+              fi
 
         - name: Save zookeeper Cache
           if: steps.restore.outputs.cache-hit != 'true'

.github/actions/tpchDataWriter/action.yml

@@ -9,10 +9,39 @@ inputs:
 runs:
     using: "composite" # Mandatory parameter
     steps:
+        - name: Start Old IGinX
+          shell: bash
+          run: |
+              cd IGinX/core/target/iginx-core-${VERSION}
+              pwd
+              export IGINX_HOME=$PWD
+              echo "IGinX home path: $IGINX_HOME"
+              cd ..
+              chmod +x iginx-core-${VERSION}/sbin/start_iginx.sh
+              nohup iginx-core-${VERSION}/sbin/start_iginx.sh > ../../iginx-${VERSION}.log 2>&1 &
+
+        # two IGinXs are using different ZKs, we need to insert data & register UDFs in both nodes
+        - name: Insert Data and Register UDF in Old IGinX
+          shell: bash
+          run: |
+              cd IGinX
+              mvn test -q -Dtest=TPCHDataGeneratorIT -DfailIfNoTests=false -P-format
+
+        - name: Show Old IGinX log
+          if: always()
+          shell: bash
+          run: cat IGinX/iginx-*.log
+
+        - name: Stop Old IGinX
+          uses: ./.github/actions/iginxRunner
+          with:
+              version: ${VERSION}
+              if-stop: "true"
+
         - name: Start New IGinX
           uses: ./.github/actions/iginxRunner
 
-        - name: Insert Data and Register UDF
+        - name: Insert Data and Register UDF in new IGinX
           shell: bash
           run: mvn test -q -Dtest=TPCHDataGeneratorIT -DfailIfNoTests=false -P-format