GH-48277: [C++][Parquet] unpack with shuffle algorithm #56460
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: R | |
| on: [push, pull_request] | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| ARCHERY_DEBUG: 1 | |
| DOCKER_VOLUME_PREFIX: ".docker/" | |
| jobs: | |
| windows-cpp: | |
| name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }} | |
| runs-on: windows-2022 | |
| if: ${{ !contains(github.event.pull_request.title, 'WIP') }} | |
| timeout-minutes: 90 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - { rtools: 40, arch: 'ucrt64' } | |
| steps: | |
| - run: git config --global core.autocrlf false | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: r-lib/actions/setup-r@v2 | |
| with: | |
| # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ | |
| # static library. The R is not used here but R 4.1 was the last R to use | |
| # Rtools40. | |
| r-version: "4.1" | |
| rtools-version: ${{ matrix.config.rtools }} | |
| Ncpus: 2 | |
| - name: Build Arrow C++ | |
| shell: bash | |
| env: | |
| MINGW_ARCH: ${{ matrix.config.arch }} | |
| RTOOLS_VERSION: ${{ matrix.config.rtools }} | |
| run: ci/scripts/r_windows_build.sh | |
| - name: Rename libarrow.zip | |
| # So that they're unique when multiple are downloaded in the next step | |
| shell: bash | |
| run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip | |
| - uses: actions/upload-artifact@v6 | |
| with: | |
| name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip | |
| path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip | |
| windows-r: | |
| needs: [windows-cpp] | |
| name: AMD64 Windows R ${{ matrix.config.rversion }} | |
| runs-on: windows-2022 | |
| if: ${{ !contains(github.event.pull_request.title, 'WIP') }} | |
| timeout-minutes: 75 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - { rversion: "release" } | |
| env: | |
| ARROW_R_CXXFLAGS: "-Werror -g" | |
| _R_CHECK_TESTS_NLINES_: 0 | |
| steps: | |
| - run: git config --global core.autocrlf false | |
| - name: Checkout Arrow | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Enable crash dumps (WER) | |
| shell: pwsh | |
| run: | | |
| New-Item -Path "C:\CrashDumps" -ItemType Directory -Force | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpFolder /t REG_EXPAND_SZ /d "C:\CrashDumps" /f | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpType /t REG_DWORD /d 2 /f | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpCount /t REG_DWORD /d 10 /f | |
| reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting" /v DontShowUI /t REG_DWORD /d 1 /f | |
| - name: Install ProcDump | |
| shell: pwsh | |
| run: | | |
| Invoke-WebRequest -Uri "https://download.sysinternals.com/files/Procdump.zip" -OutFile "$env:TEMP\procdump.zip" | |
| Expand-Archive -Path "$env:TEMP\procdump.zip" -DestinationPath "C:\procdump" -Force | |
| # Install as AeDebug postmortem debugger (second-chance, for non-R processes) | |
| C:\procdump\procdump64.exe -accepteula -ma -i C:\CrashDumps | |
| # This must be done before r-lib/actions/setup-r because curl in | |
| # Rtools doesn't work on non Rtools' MSYS2 environment. If we | |
| # use "shell: bash" after r-lib/actions/setup-r, bash in Rtools | |
| # is used on non Rtools' MSYS2 environment. | |
| - name: Install MinIO | |
| shell: bash | |
| run: | | |
| mkdir -p "$HOME/.local/bin" | |
| ci/scripts/install_minio.sh latest "$HOME/.local" | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: mkdir r/windows | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: libarrow-rtools40-ucrt64.zip | |
| path: r/windows | |
| - name: Unzip and rezip libarrows | |
| shell: bash | |
| run: | | |
| cd r/windows | |
| ls *.zip | xargs -n 1 unzip -uo | |
| rm -rf *.zip | |
| - uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: ${{ matrix.config.rversion }} | |
| Ncpus: 2 | |
| - uses: r-lib/actions/setup-r-dependencies@v2 | |
| env: | |
| GITHUB_PAT: "${{ github.token }}" | |
| with: | |
| # For some arcane reason caching does not work on the windows runners | |
| # most likely due to https://github.com/actions/cache/issues/815 | |
| cache: false | |
| working-directory: 'r' | |
| extra-packages: | | |
| any::rcmdcheck | |
| any::cyclocomp | |
| # TODO(ARROW-17149): figure out why the GCS tests are hanging on Windows | |
| # - name: Install Google Cloud Storage Testbench | |
| # shell: bash | |
| # run: ci/scripts/install_gcs_testbench.sh default | |
| - name: Check | |
| shell: Rscript {0} | |
| run: | | |
| # Because we do R CMD build and r/windows is in .Rbuildignore, | |
| # assemble the libarrow.zip file and pass it as an env var | |
| setwd("r/windows") | |
| zip("libarrow.zip", ".") | |
| setwd("..") | |
| Sys.setenv( | |
| RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), | |
| MAKEFLAGS = paste0("-j", parallel::detectCores()), | |
| ARROW_R_DEV = TRUE, | |
| "_R_CHECK_FORCE_SUGGESTS_" = FALSE, | |
| "_R_CHECK_STOP_ON_INVALID_NUMERIC_VERSION_INPUTS_" = TRUE, | |
| "_R_CHECK_DONTTEST_EXAMPLES_" = TRUE | |
| ) | |
| rcmdcheck::rcmdcheck(".", | |
| build_args = '--no-build-vignettes', | |
| args = c('--no-manual', '--as-cran', '--ignore-vignettes'), | |
| error_on = 'warning', | |
| check_dir = 'check', | |
| timeout = 3600 | |
| ) | |
| - name: Dump install logs | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| if (Test-Path "r/check/arrow.Rcheck/00install.out") { | |
| Get-Content "r/check/arrow.Rcheck/00install.out" | |
| } else { | |
| Write-Host "00install.out not found" | |
| } | |
| - name: Dump test logs | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| $files = Get-ChildItem -Path "r/check" -Recurse -Filter "testthat.Rout*" -ErrorAction SilentlyContinue | |
| if ($files) { | |
| foreach ($f in $files) { | |
| Write-Host "=== $($f.FullName) ===" | |
| Get-Content $f.FullName | |
| } | |
| } else { | |
| Write-Host "No testthat.Rout files found" | |
| } | |
| - name: Dump check log | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| $files = Get-ChildItem -Path "r/check" -Recurse -Filter "00check.log" -ErrorAction SilentlyContinue | |
| if ($files) { | |
| foreach ($f in $files) { | |
| Write-Host "=== $($f.FullName) ===" | |
| Get-Content $f.FullName | |
| } | |
| } else { | |
| Write-Host "No 00check.log found" | |
| } | |
| - name: List check directory contents | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| if (Test-Path "r/check") { | |
| Get-ChildItem -Path "r/check" -Recurse | Format-Table FullName, Length, LastWriteTime | |
| } else { | |
| Write-Host "r/check directory not found" | |
| } | |
| - name: Query crash events from Event Log | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| Write-Host "=== Application Error (Event ID 1000) ===" | |
| Get-WinEvent -FilterHashtable @{LogName='Application'; ProviderName='Application Error'} -MaxEvents 10 -ErrorAction SilentlyContinue | | |
| Format-List TimeCreated, Message | |
| Write-Host "=== Windows Error Reporting (Event ID 1001) ===" | |
| Get-WinEvent -FilterHashtable @{LogName='Application'; ProviderName='Windows Error Reporting'} -MaxEvents 10 -ErrorAction SilentlyContinue | | |
| Format-List TimeCreated, Message | |
| Write-Host "=== Application Crashes and Hangs ===" | |
| Get-WinEvent -FilterHashtable @{LogName='Application'; Level=1,2} -MaxEvents 20 -ErrorAction SilentlyContinue | | |
| Format-List TimeCreated, ProviderName, Id, LevelDisplayName, Message | |
| - name: List crash dumps | |
| if: always() | |
| shell: pwsh | |
| run: | | |
| if (Test-Path "C:\CrashDumps") { | |
| Get-ChildItem "C:\CrashDumps" -Recurse | Format-Table Name, Length, LastWriteTime | |
| } else { | |
| Write-Host "No crash dump directory found" | |
| } | |
| - name: Upload crash dumps | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: crash-dumps | |
| path: C:\CrashDumps\ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Upload test parquet files | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test-parquet-files | |
| path: r/check/arrow.Rcheck/tests/testthat/test-n_distinct-many-batches.parquet | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Upload full check directory | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: check-output | |
| path: r/check/ | |
| if-no-files-found: ignore | |
| retention-days: 7 |