GH-48277: [C++][Parquet] unpack with shuffle algorithm #56459
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: R | |
| on: [push, pull_request] | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| ARCHERY_DEBUG: 1 | |
| DOCKER_VOLUME_PREFIX: ".docker/" | |
| jobs: | |
| windows-cpp: | |
| name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }} | |
| runs-on: windows-2022 | |
| if: ${{ !contains(github.event.pull_request.title, 'WIP') }} | |
| timeout-minutes: 90 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - { rtools: 40, arch: 'ucrt64' } | |
| steps: | |
| - run: git config --global core.autocrlf false | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: r-lib/actions/setup-r@v2 | |
| with: | |
| # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ | |
| # static library. The R is not used here but R 4.1 was the last R to use | |
| # Rtools40. | |
| r-version: "4.1" | |
| rtools-version: ${{ matrix.config.rtools }} | |
| Ncpus: 2 | |
| - name: Build Arrow C++ | |
| shell: bash | |
| env: | |
| MINGW_ARCH: ${{ matrix.config.arch }} | |
| RTOOLS_VERSION: ${{ matrix.config.rtools }} | |
| run: ci/scripts/r_windows_build.sh | |
| - name: Rename libarrow.zip | |
| # So that they're unique when multiple are downloaded in the next step | |
| shell: bash | |
| run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip | |
| - uses: actions/upload-artifact@v6 | |
| with: | |
| name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip | |
| path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip | |
| windows-r: | |
| needs: [windows-cpp] | |
| name: AMD64 Windows R ${{ matrix.config.rversion }} | |
| runs-on: windows-2022 | |
| if: ${{ !contains(github.event.pull_request.title, 'WIP') }} | |
| timeout-minutes: 75 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - { rversion: "release" } | |
| env: | |
| ARROW_R_CXXFLAGS: "-Werror -g" | |
| _R_CHECK_TESTS_NLINES_: 0 | |
| steps: | |
| - run: git config --global core.autocrlf false | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Enable crash dumps (WER) | |
| shell: pwsh | |
| run: | | |
| New-Item -Path "C:\CrashDumps" -ItemType Directory -Force | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpFolder /t REG_EXPAND_SZ /d "C:\CrashDumps" /f | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpType /t REG_DWORD /d 2 /f | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpCount /t REG_DWORD /d 10 /f | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting" /v DontShowUI /t REG_DWORD /d 1 /f | |
| - name: Install ProcDump | |
| shell: pwsh | |
| run: | | |
| Invoke-WebRequest -Uri "https://download.sysinternals.com/files/Procdump.zip" -OutFile "$env:TEMP\procdump.zip" | |
| Expand-Archive -Path "$env:TEMP\procdump.zip" -DestinationPath "C:\procdump" -Force | |
| # Install as AeDebug postmortem debugger (second-chance, for non-R processes) | |
| C:\procdump\procdump64.exe -accepteula -ma -i C:\CrashDumps | |
| # This must be done before r-lib/actions/setup-r because curl in | |
| # Rtools doesn't work on non Rtools' MSYS2 environment. If we | |
| # use "shell: bash" after r-lib/actions/setup-r, bash in Rtools | |
| # is used on non Rtools' MSYS2 environment. | |
| - name: Install MinIO | |
| shell: bash | |
| run: | | |
| mkdir -p "$HOME/.local/bin" | |
| ci/scripts/install_minio.sh latest "$HOME/.local" | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: mkdir r/windows | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: libarrow-rtools40-ucrt64.zip | |
| path: r/windows | |
| - name: Unzip and rezip libarrows | |
| shell: bash | |
| run: | | |
| cd r/windows | |
| ls *.zip | xargs -n 1 unzip -uo | |
| rm -rf *.zip | |
| - uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: ${{ matrix.config.rversion }} | |
| Ncpus: 2 | |
| - uses: r-lib/actions/setup-r-dependencies@v2 | |
| env: | |
| GITHUB_PAT: "${{ github.token }}" | |
| with: | |
| # For some arcane reason caching does not work on the windows runners | |
| # most likely due to https://github.com/actions/cache/issues/815 | |
| cache: false | |
| working-directory: 'r' | |
| extra-packages: | | |
| any::rcmdcheck | |
| any::cyclocomp | |
| # TODO(ARROW-17149): figure out why the GCS tests are hanging on Windows | |
| # - name: Install Google Cloud Storage Testbench | |
| # shell: bash | |
| # run: ci/scripts/install_gcs_testbench.sh default | |
| - name: Prepare libarrow.zip | |
| shell: Rscript {0} | |
| run: | | |
| # Because we do R CMD build and r/windows is in .Rbuildignore, | |
| # assemble the libarrow.zip file and pass it as an env var | |
| setwd("r/windows") | |
| zip("libarrow.zip", ".") | |
| - name: Enable IFEO crash monitoring for R processes | |
| shell: pwsh | |
| run: | | |
| # Use Image File Execution Options to auto-attach ProcDump to EVERY instance | |
| # of Rterm.exe and R.exe. This catches first-chance exceptions (before R's own | |
| # SEH handler swallows them) across ALL child processes spawned by R CMD check. | |
| # -e 1 = first-chance, -f = filter to: access violation, illegal instruction, | |
| # stack overflow, heap corruption, integer divide by zero, privileged instruction | |
| $procdumpArgs = '"C:\procdump\procdump64.exe" -accepteula -ma -e 1 -f C0000005 -f C000001D -f C00000FD -f C0000374 -f C0000094 -f C0000096 -x C:\CrashDumps' | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\Rterm.exe" /v Debugger /t REG_SZ /d $procdumpArgs /f | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\R.exe" /v Debugger /t REG_SZ /d $procdumpArgs /f | |
| - name: Check | |
| shell: Rscript {0} | |
| run: | | |
| setwd("r") | |
| Sys.setenv( | |
| RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), | |
| MAKEFLAGS = paste0("-j", parallel::detectCores()), | |
| ARROW_R_DEV = TRUE, | |
| "_R_CHECK_FORCE_SUGGESTS_" = FALSE, | |
| "_R_CHECK_STOP_ON_INVALID_NUMERIC_VERSION_INPUTS_" = TRUE, | |
| "_R_CHECK_DONTTEST_EXAMPLES_" = TRUE | |
| ) | |
| # IFEO auto-attaches ProcDump to all R.exe/Rterm.exe child processes | |
| # so first-chance exceptions (segfault, illegal instruction, etc.) are | |
| # caught and dumped to C:\CrashDumps before R's own handler swallows them. | |
| rcmdcheck::rcmdcheck(".", | |
| build_args = '--no-build-vignettes', | |
| args = c('--no-manual', '--as-cran', '--ignore-vignettes'), | |
| error_on = 'warning', | |
| check_dir = 'check', | |
| timeout = 3600 | |
| ) | |
| - name: Dump install logs | |
| shell: cmd | |
| run: cat r/check/arrow.Rcheck/00install.out | |
| if: always() | |
| - name: Dump test logs | |
| shell: bash | |
| run: find r/check -name 'testthat.Rout*' -exec cat '{}' \; || true | |
| if: always() | |
| - name: Query crash events from Event Log | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| Write-Host "=== Application Crashes (Event ID 1000) ===" | |
| Get-WinEvent -FilterHashtable @{LogName='Application'; Id=1000} -MaxEvents 20 -ErrorAction SilentlyContinue | | |
| Format-List TimeCreated, Message | |
| Write-Host "=== WER Reports (Event ID 1001) ===" | |
| Get-WinEvent -FilterHashtable @{LogName='Application'; Id=1001} -MaxEvents 20 -ErrorAction SilentlyContinue | | |
| Format-List TimeCreated, Message | |
| - name: List crash dumps | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| if (Test-Path "C:\CrashDumps") { | |
| Get-ChildItem "C:\CrashDumps" -Recurse | Format-Table Name, Length, LastWriteTime | |
| } else { | |
| Write-Host "No crash dump directory found" | |
| } | |
| - name: Upload crash dumps | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: crash-dumps | |
| path: C:\CrashDumps\ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Upload test parquet files | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test-parquet-files | |
| path: r/check/arrow.Rcheck/tests/testthat/test-n_distinct-many-batches.parquet | |
| if-no-files-found: ignore | |
| retention-days: 7 |