Merge branch 'iejoin-parallel' into iejoin-lsorts

Author: hawkfish

.github/actions/cleanup_runner/action.yml

@@ -19,4 +19,3 @@ runs:
     - name: After
       shell: bash
       run: df -h
-

.github/config/extensions/azure.cmake

@@ -1,7 +1,7 @@
 if (NOT MINGW AND NOT ${WASM_ENABLED})
-    duckdb_extension_load(azure
-            LOAD_TESTS
-            GIT_URL https://github.com/duckdb/duckdb-azure
-            GIT_TAG 0709c0fa1cf67a668b58b1f06ff3e5fc1696e10a
-            )
+  duckdb_extension_load(azure
+        LOAD_TESTS
+        GIT_URL https://github.com/duckdb/duckdb-azure
+        GIT_TAG f4d55b658c14d3de40bac9d1c68f08f3cf116374
+  )
 endif()

.github/config/extensions/delta.cmake

@@ -1,7 +1,7 @@
 if (NOT MINGW AND NOT ${WASM_ENABLED})
     duckdb_extension_load(delta
             GIT_URL https://github.com/duckdb/duckdb-delta
-            GIT_TAG 6515bb2560772956f9b74a18a47782f18ed4d827
+            GIT_TAG 50de51108cc4d2c09a549e022fce4f74e17bf360
             SUBMODULES extension-ci-tools
             APPLY_PATCHES
     )

.github/config/extensions/spatial.cmake

@@ -3,7 +3,7 @@ if (${BUILD_COMPLETE_EXTENSION_SET})
 duckdb_extension_load(spatial
     DONT_LINK LOAD_TESTS
     GIT_URL https://github.com/duckdb/duckdb-spatial
-    GIT_TAG d83faf88cd7e4d4cca4edd056a530382c5018654
+    GIT_TAG 2f2668d211c0cf759f460403a108f24eb8b887e3
     INCLUDE_DIR src/spatial
     TEST_DIR test/sql
     APPLY_PATCHES

.github/config/extensions/unity_catalog.cmake

@@ -0,0 +1,7 @@
+if (NOT MINGW AND NOT ${WASM_ENABLED} AND NOT ${MUSL_ENABLED})
+    duckdb_extension_load(unity_catalog
+            GIT_URL https://github.com/duckdb/unity_catalog
+            GIT_TAG 1ad4f0b1fb12f661ff58fc91c2c9e7022591ea4b
+            LOAD_TESTS
+            )
+endif()

.github/workflows/Extensions.yml

@@ -19,6 +19,10 @@ on:
         type: boolean
         required: false
         default: true
+      opt_in_archs:
+        type: string
+        required: false
+        default: ''
   workflow_dispatch:
     inputs:
       override_git_describe:
@@ -34,6 +38,11 @@ on:
         type: string
         required: false
         default: ''
+      opt_in_archs:
+        description: 'Semicolon-separated list of architectures to opt into'
+        type: string
+        required: false
+        default: 'windows_arm64;'
       skip_tests:
         description: 'Set to true to skip all testing'
         type: boolean
@@ -96,6 +105,7 @@ jobs:
     outputs:
       main_extensions_config: ${{ steps.set-main-extensions.outputs.extension_config }}
       main_extensions_exclude_archs: ${{ steps.set-main-extensions.outputs.exclude_archs }}
+      main_extensions_opt_in_archs: ${{ steps.set-main-extensions.outputs.opt_in_archs }}
       rust_based_extensions_config: ${{ steps.set-rust-based-extensions.outputs.extension_config }}
       rust_based_extensions_exclude_archs: ${{ steps.set-rust-based-extensions.outputs.exclude_archs }}
       external_extensions_config: ${{ steps.set-external-extensions.outputs.extension_config }}
@@ -105,6 +115,7 @@ jobs:
       # NOTE: on PRs we exclude some archs to speed things up
       BASE_EXCLUDE_ARCHS: ${{ (github.event_name == 'pull_request' || inputs.run_all != 'true') && 'wasm_mvp;wasm_threads;windows_amd64_mingw;osx_amd64;linux_arm64;linux_amd64_musl;' || '' }}
       EXTRA_EXCLUDE_ARCHS: ${{ inputs.extra_exclude_archs }}
+      OPT_IN_ARCHS: ${{ inputs.opt_in_archs }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -120,6 +131,7 @@ jobs:
         run: |
           # Set config
           echo exclude_archs="$DEFAULT_EXCLUDE_ARCHS;$BASE_EXCLUDE_ARCHS;$EXTRA_EXCLUDE_ARCHS" >> $GITHUB_OUTPUT
+          echo opt_in_archs="$OPT_IN_ARCHS" >> $GITHUB_OUTPUT
           in_tree_extensions="`cat $IN_TREE_CONFIG_FILE`"
           out_of_tree_extensions="`cat $OUT_OF_TREE_CONFIG_FILE`"
           echo "extension_config<<EOF" >> $GITHUB_OUTPUT
@@ -164,6 +176,7 @@ jobs:
     with:
       artifact_prefix: main-extensions
       exclude_archs: ${{ needs.load-extension-configs.outputs.main_extensions_exclude_archs }}
+      opt_in_archs: ${{ needs.load-extension-configs.outputs.main_extensions_opt_in_archs }}
       extension_config: ${{ needs.load-extension-configs.outputs.main_extensions_config }}
       override_tag: ${{ inputs.override_git_describe }}
       override_duckdb_version: ${{ inputs.git_ref }}
@@ -215,22 +228,24 @@ jobs:
         with:
           fetch-depth: 0
 
+      - uses: ./.github/actions/cleanup_runner
+
       - uses: actions/download-artifact@v4
         name: Download main extensions
         with:
-          pattern: main-extensions-${{ github.sha }}*
+          pattern: main-extensions*
           path: /tmp/repository_generation/main-extensions
 
       - uses: actions/download-artifact@v4
         name: Download rust-based extensions
         with:
-          pattern: rust-based-extensions-${{ github.sha }}*
+          pattern: rust-based-extensions*
           path: /tmp/repository_generation/rust-based-extensions
 
       - uses: actions/download-artifact@v4
         name: Download external extensions
         with:
-          pattern: external-extensions-${{ github.sha }}*
+          pattern: external-extensions*
           path: /tmp/repository_generation/external-extensions
 
       - name: Print all extensions
@@ -246,7 +261,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           if-no-files-found: error
-          name: extension-repository-${{ github.sha }}
+          name: extension-repository
           path: |
             /tmp/merged_repository/**/*.duckdb_extension*
 
@@ -266,13 +281,13 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          pattern: extension-repository-${{ github.sha }}
+          pattern: extension-repository
           path: /tmp
 
       - name: List extensions to deploy
         shell: bash
         run: |
-          tree /tmp/extension-repository-${{ github.sha }}
+          tree /tmp/extension-repository
 
       - name: Deploy extensions
         shell: bash
@@ -284,7 +299,7 @@ jobs:
           DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
         run: |
           pip install awscli
-          ./scripts/extension-upload-repository.sh /tmp/extension-repository-${{ github.sha }}
+          ./scripts/extension-upload-repository.sh /tmp/extension-repository
 
   autoload-tests:
     name: Extension Autoloading Tests
@@ -311,13 +326,13 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          pattern: extension-repository-${{ github.sha }}
+          pattern: extension-repository
           path: /tmp
 
       - name: List extensions to test with
         shell: bash
         run: |
-          tree /tmp/extension-repository-${{ github.sha }}
+          tree /tmp/extension-repository
 
       - name: Build DuckDB
         env:
@@ -333,7 +348,7 @@ jobs:
 
       - name: Run Tests
         env:
-          LOCAL_EXTENSION_REPO: /tmp/extension-repository-${{ github.sha }}
+          LOCAL_EXTENSION_REPO: /tmp/extension-repository
         run: |
           ./build/release/test/unittest --autoloading available --skip-compiled
 
@@ -378,13 +393,13 @@ jobs:
       - uses: actions/download-artifact@v4
         name: Download extension repository artifact
         with:
-          pattern: extension-repository-${{ github.sha }}
+          pattern: extension-repository
           path: /tmp
 
       - name: Copy over local extension repository
         shell: bash
         run: |
-          cp -r /tmp/extension-repository-${{ github.sha }} build/release/repository
+          cp -r /tmp/extension-repository build/release/repository
           tree build/release/repository
           find build/release/repository -type f ! -path "build/release/repository/*/linux_amd64/*" -delete
           tree build/release/repository

.github/workflows/InvokeCI.yml

@@ -27,6 +27,7 @@ jobs:
       git_ref: ${{ inputs.git_ref }}
       skip_tests: ${{ inputs.skip_tests }}
       run_all: ${{ inputs.run_all }}
+      opt_in_archs: 'windows_arm64;'
 
   osx:
     uses: ./.github/workflows/OSX.yml

.github/workflows/_extension_distribution.yml

@@ -12,6 +12,10 @@ on:
         required: false
         type: string
         default: ''
+      opt_in_archs:
+        required: false
+        type: string
+        default: ''
       extra_toolchains:
         required: false
         type: string
@@ -55,6 +59,7 @@ jobs:
       ci_tools_version: main
 
       exclude_archs: ${{ inputs.exclude_archs }}
+      opt_in_archs: ${{ inputs.opt_in_archs }}
 
       extra_toolchains: ${{ inputs.extra_toolchains }}
       use_merged_vcpkg_manifest: '1'

.sanitizer-thread-suppressions.txt

@@ -6,5 +6,4 @@ race:duckdb_moodycamel
 race:*duckdb/extension/jemalloc/jemalloc/*
 race:AddToEvictionQueue
 race:ValidityAppend
-race:GetPayload
-race:SetPayload
+race:ResolveComplexJoin

data/19578.csv

@@ -0,0 +1,2 @@
+Country;Location;Count;Date
+China;"35.86166;104.19397";97100,7;04/07/20