diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 0000000..b5b9114 --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,142 @@ +name: Build +description: Build yek across different platforms + +inputs: + upload_artifacts: + required: false + default: "false" + description: "Whether to upload artifacts" + +outputs: + binary_path: + description: "Path to the built binary" + value: ${{ steps.get_binary_path.outputs.path }} + +runs: + using: "composite" + steps: + - name: Install OpenSSL (Ubuntu) + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev + + - name: Install musl tools (Linux musl) + if: runner.os == 'Linux' && contains(matrix.target, 'musl') + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y musl-tools musl-dev + # For aarch64 target + if [[ "${{ matrix.target }}" == "aarch64"* ]]; then + # Install musl cross-compiler + sudo apt-get install -y musl-dev musl-tools + git clone https://github.com/richfelker/musl-cross-make.git + cd musl-cross-make + echo "TARGET = aarch64-linux-musl" > config.mak + echo "OUTPUT = /usr/local" >> config.mak + make -j$(nproc) + sudo make install + cd .. + rm -rf musl-cross-make + fi + + - name: Install OpenSSL (macOS) + if: runner.os == 'macOS' + shell: bash + run: | + brew install openssl@3 + echo "OPENSSL_DIR=$(brew --prefix openssl@3)" >> $GITHUB_ENV + echo "PKG_CONFIG_PATH=$(brew --prefix openssl@3)/lib/pkgconfig" >> $GITHUB_ENV + + - name: Install OpenSSL (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + choco install openssl --params='/InstallationPath:C:\OpenSSL\' + echo "OPENSSL_DIR=C:\OpenSSL" | Out-File -FilePath $env:GITHUB_ENV -Append + echo "PKG_CONFIG_PATH=C:\OpenSSL\lib\pkgconfig" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Install cross (Linux) + if: ${{ contains(matrix.target, 'linux') && contains(matrix.target, 'gnu') }} + shell: bash + run: cargo install cross + + - name: Setup Rust (Native builds) + if: ${{ !contains(matrix.target, 'linux') || !contains(matrix.target, 'gnu') }} + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt,clippy + + - name: Install Rust target + if: ${{ !contains(matrix.target, 'linux') || !contains(matrix.target, 'gnu') }} + shell: bash + run: rustup target add ${{ matrix.target }} + + - name: Build with cross (Linux GNU) + if: ${{ contains(matrix.target, 'linux') && contains(matrix.target, 'gnu') }} + shell: bash + run: cross build --release --target ${{ matrix.target }} + + - name: Build with cross (Linux MUSL) + if: ${{ contains(matrix.target, 'linux') && contains(matrix.target, 'musl') }} + shell: bash + run: | + # Set CC and other flags for musl builds + if [[ "${{ matrix.target }}" == "aarch64"* ]]; then + export CC=aarch64-linux-musl-gcc + export AR=aarch64-linux-musl-ar + export RUSTFLAGS="-C linker=aarch64-linux-musl-gcc" + export PKG_CONFIG_ALLOW_CROSS=1 + export OPENSSL_STATIC=1 + # Configure pkg-config for cross-compilation + export PKG_CONFIG_SYSROOT_DIR=/usr/local/aarch64-linux-musl + export PKG_CONFIG_PATH=/usr/local/aarch64-linux-musl/lib/pkgconfig + # Build OpenSSL statically for aarch64-musl + git clone https://github.com/openssl/openssl.git + cd openssl + ./Configure linux-aarch64 --prefix=/usr/local/aarch64-linux-musl no-shared + make -j$(nproc) + sudo make install + cd .. + rm -rf openssl + else + export CC=musl-gcc + fi + cargo build --release --target ${{ matrix.target }} + + - name: Native build (macOS/Windows) + if: ${{ !contains(matrix.target, 'linux') }} + shell: bash + run: cargo build --release --target ${{ matrix.target }} + + - name: Get binary path (Unix) + if: runner.os != 'Windows' + id: unix_path + shell: bash + run: echo "path=target/${{ matrix.target }}/release/${{ matrix.artifact_name }}" >> $GITHUB_OUTPUT + + - name: Get binary path (Windows) + if: runner.os == 'Windows' + id: windows_path + shell: pwsh + run: echo "path=target\${{ matrix.target }}\release\${{ matrix.artifact_name }}" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + + - name: Set final path + id: get_binary_path + shell: bash + run: | + if [ "${{ runner.os }}" = "Windows" ]; then + echo "path=${{ steps.windows_path.outputs.path }}" >> $GITHUB_OUTPUT + else + echo "path=${{ steps.unix_path.outputs.path }}" >> $GITHUB_OUTPUT + fi + + - name: Upload artifact + if: ${{ inputs.upload_artifacts == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.target }}-${{ matrix.asset_name }} + path: ${{ steps.get_binary_path.outputs.path }} + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb6897e..cdd3110 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,22 +76,18 @@ jobs: # Linux builds using cross - os: ubuntu-latest target: x86_64-unknown-linux-gnu - use-cross: true artifact_name: yek asset_name: yek-x86_64-unknown-linux-gnu.tar.gz - os: ubuntu-latest target: aarch64-unknown-linux-gnu - use-cross: true artifact_name: yek asset_name: yek-aarch64-unknown-linux-gnu.tar.gz - os: ubuntu-latest target: x86_64-unknown-linux-musl - use-cross: true artifact_name: yek asset_name: yek-x86_64-unknown-linux-musl.tar.gz - os: ubuntu-latest target: aarch64-unknown-linux-musl - use-cross: true artifact_name: yek asset_name: yek-aarch64-unknown-linux-musl.tar.gz @@ -118,53 +114,54 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install cross (Linux) - if: matrix.use-cross - run: cargo install cross - - - name: Setup Rust (Native builds) - if: ${{ !matrix.use-cross }} - uses: actions-rust-lang/setup-rust-toolchain@v1 + - uses: ./.github/actions/build + id: build with: - components: rustfmt,clippy - - - name: Install Rust target - if: ${{ !matrix.use-cross }} - run: rustup target add ${{ matrix.target }} - - - name: Build with cross (Linux) - if: matrix.use-cross - run: cross build --release --target ${{ matrix.target }} - - - name: Native build (macOS/Windows) - if: ${{ !matrix.use-cross }} - run: cargo build --release --target ${{ matrix.target }} + upload_artifacts: "false" - - name: Package + - name: Package binary (Unix) + if: runner.os != 'Windows' shell: bash run: | mkdir -p release-artifacts staging="yek-${{ matrix.target }}" mkdir -p "$staging" - if [[ "${{ runner.os }}" == "Windows" ]]; then - cp "target/${{ matrix.target }}/release/${{ matrix.artifact_name }}" "$staging/" - 7z a "release-artifacts/${{ matrix.asset_name }}" "$staging" - else - cp "target/${{ matrix.target }}/release/${{ matrix.artifact_name }}" "$staging/" - tar czf "release-artifacts/${{ matrix.asset_name }}" "$staging" - fi + cp "${{ steps.build.outputs.binary_path }}" "$staging/" + tar czf "release-artifacts/${{ matrix.asset_name }}" "$staging" # Verify the artifact exists and has size > 0 - if [[ ! -f "release-artifacts/${{ matrix.asset_name }}" ]]; then + if [ ! -f "release-artifacts/${{ matrix.asset_name }}" ]; then echo "::error::Packaged artifact not found: release-artifacts/${{ matrix.asset_name }}" exit 1 fi - if [[ ! -s "release-artifacts/${{ matrix.asset_name }}" ]]; then + if [ ! -s "release-artifacts/${{ matrix.asset_name }}" ]; then echo "::error::Packaged artifact is empty: release-artifacts/${{ matrix.asset_name }}" exit 1 fi echo "Successfully packaged ${{ matrix.asset_name }}" ls -lh "release-artifacts/${{ matrix.asset_name }}" + - name: Package binary (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path release-artifacts + $staging = "yek-${{ matrix.target }}" + New-Item -ItemType Directory -Force -Path $staging + Copy-Item "${{ steps.build.outputs.binary_path }}" -Destination $staging + 7z a "release-artifacts\${{ matrix.asset_name }}" "$staging" + # Verify the artifact exists and has size > 0 + if (-not (Test-Path "release-artifacts\${{ matrix.asset_name }}")) { + Write-Error "Packaged artifact not found: release-artifacts\${{ matrix.asset_name }}" + exit 1 + } + $fileInfo = Get-Item "release-artifacts\${{ matrix.asset_name }}" + if ($fileInfo.Length -eq 0) { + Write-Error "Packaged artifact is empty: release-artifacts\${{ matrix.asset_name }}" + exit 1 + } + Write-Host "Successfully packaged ${{ matrix.asset_name }}" + Get-Item "release-artifacts\${{ matrix.asset_name }}" | Select-Object Length,LastWriteTime + - name: Upload artifact uses: actions/upload-artifact@v4 with: @@ -172,70 +169,6 @@ jobs: path: release-artifacts/${{ matrix.asset_name }} if-no-files-found: error - benchmark: - name: Benchmark / ${{ matrix.benchmark_group.name }} - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - strategy: - matrix: - benchmark_group: - - group: "SingleFile_ByteMode" - name: "Single File Byte Mode" - - group: "SingleFile_ByteMode_Large" - name: "Single File Byte Mode Large" - - group: "SingleFile_TokenMode_Large" - name: "Single File Token Mode Large" - - group: "MultipleFiles_Small" - name: "Multiple Files Small" - - group: "MultipleFiles_Medium" - name: "Multiple Files Medium" - - group: "MultipleFiles_Large" - name: "Multiple Files Large" - - group: "MultipleFiles_TokenMode" - name: "Multiple Files Token Mode" - - group: "CustomConfig" - name: "Custom Config" - fail-fast: false - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - - - name: Build benchmarks on target branch - run: | - git fetch origin ${{ github.base_ref }} - git checkout ${{ github.base_ref }} - cargo bench --bench serialization --no-run - - - name: Run benchmark on target branch - run: cargo bench --bench serialization -- --save-baseline ${{ github.base_ref }} '${{ matrix.benchmark_group.group }}/' - - - name: Build benchmarks on PR branch - run: | - git checkout ${{ github.head_ref }} - cargo bench --bench serialization --no-run - - - name: Compare benchmarks - run: | - cargo bench --bench serialization -- --baseline ${{ github.base_ref }} --noise-threshold 2 '${{ matrix.benchmark_group.group }}/' > benchmark_results.md - echo "## Benchmark Results for ${{ matrix.benchmark_group.name }}" >> $GITHUB_STEP_SUMMARY - cat benchmark_results.md >> $GITHUB_STEP_SUMMARY - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - with: - name: criterion-${{ matrix.benchmark_group.group }}-results - path: benchmark_results.md - if-no-files-found: error - release: name: Release needs: [test, lint, build] diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 0000000..5306309 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,158 @@ +name: Perf + +on: + push: + branches: [main] + pull_request: + branches: [main] + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: short + OPENSSL_STATIC: true + PKG_CONFIG_ALLOW_CROSS: true + +jobs: + stress: + name: Stress ${{ matrix.target }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # Linux builds using cross + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + artifact_name: yek + asset_name: yek-x86_64-unknown-linux-gnu.tar.gz + - os: ubuntu-latest + target: aarch64-unknown-linux-gnu + artifact_name: yek + asset_name: yek-aarch64-unknown-linux-gnu.tar.gz + - os: ubuntu-latest + target: x86_64-unknown-linux-musl + artifact_name: yek + asset_name: yek-x86_64-unknown-linux-musl.tar.gz + - os: ubuntu-latest + target: aarch64-unknown-linux-musl + artifact_name: yek + asset_name: yek-aarch64-unknown-linux-musl.tar.gz + + # Native macOS builds + - os: macos-latest + target: x86_64-apple-darwin + artifact_name: yek + asset_name: yek-x86_64-apple-darwin.tar.gz + - os: macos-latest + target: aarch64-apple-darwin + artifact_name: yek + asset_name: yek-aarch64-apple-darwin.tar.gz + + # Native Windows builds + - os: windows-latest + target: x86_64-pc-windows-msvc + artifact_name: yek.exe + asset_name: yek-x86_64-pc-windows-msvc.zip + - os: windows-latest + target: aarch64-pc-windows-msvc + artifact_name: yek.exe + asset_name: yek-aarch64-pc-windows-msvc.zip + + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/build + id: build + with: + upload_artifacts: "false" + + - name: Install yek (Unix) + if: runner.os != 'Windows' + shell: bash + run: cargo install --path . + + - name: Install yek (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: cargo install --path . + + - name: Checkout VSCode repository + uses: actions/checkout@v4 + with: + repository: microsoft/vscode + path: vscode + fetch-depth: 1 + + - name: Run yek (Unix) + if: runner.os != 'Windows' + shell: bash + timeout-minutes: 1 + run: cd vscode && yek + + - name: Run yek (Windows) + if: runner.os == 'Windows' + shell: pwsh + timeout-minutes: 1 + run: | + Set-Location vscode + yek + + benchmark: + name: Benchmark / ${{ matrix.benchmark_group.name }} + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + benchmark_group: + - { group: "SingleFile_ByteMode", name: "Single File Byte Mode" } + - { + group: "SingleFile_ByteMode_Large", + name: "Single File Byte Mode Large", + } + - { + group: "SingleFile_TokenMode_Large", + name: "Single File Token Mode Large", + } + - { group: "MultipleFiles_Small", name: "Multiple Files Small" } + - { group: "MultipleFiles_Medium", name: "Multiple Files Medium" } + - { group: "MultipleFiles_Large", name: "Multiple Files Large" } + - { + group: "MultipleFiles_TokenMode", + name: "Multiple Files Token Mode", + } + - { group: "CustomConfig", name: "Custom Config" } + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Run benchmarks + run: | + # Build and run on target branch + git fetch origin ${{ github.base_ref }} + git checkout ${{ github.base_ref }} + cargo bench --bench serialization --no-run + cargo bench --bench serialization -- --save-baseline ${{ github.base_ref }} '${{ matrix.benchmark_group.group }}/' + + # Build and compare on PR branch + git checkout ${{ github.head_ref }} + cargo bench --bench serialization --no-run + cargo bench --bench serialization -- --baseline ${{ github.base_ref }} --noise-threshold 2 '${{ matrix.benchmark_group.group }}/' > benchmark_results.md + echo "## Benchmark Results for ${{ matrix.benchmark_group.name }}" >> $GITHUB_STEP_SUMMARY + cat benchmark_results.md >> $GITHUB_STEP_SUMMARY + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: criterion-${{ matrix.benchmark_group.group }}-results + path: benchmark_results.md + if-no-files-found: error diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 91d26a0..f6b90e7 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -1,4 +1,4 @@ ---- +# after publishing a release, run this to test the installation scripts from bodo.run are working name: Installation Test on: diff --git a/src/config.rs b/src/config.rs index 4c8e57f..9c15ceb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -78,6 +78,10 @@ pub struct YekConfig { /// Final resolved output file path (only used if not streaming) pub output_file_full_path: Option, + + /// Maximum depth to search for Git commit times + #[config_arg(accept_from = "config_only", default_value = "100")] + pub max_git_depth: i32, } /// Provide defaults so tests or other callers can create a baseline YekConfig easily. @@ -104,6 +108,7 @@ impl Default for YekConfig { stream: false, token_mode: false, output_file_full_path: None, + max_git_depth: 100, } } } diff --git a/src/lib.rs b/src/lib.rs index 91e7ecb..aaee2f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,7 +45,10 @@ pub fn serialize_repo(config: &YekConfig) -> Result<(String, Vec) .par_iter() .filter_map(|dir| { let repo_path = Path::new(dir); - priority::get_recent_commit_times_git2(repo_path) + priority::get_recent_commit_times_git2( + repo_path, + config.max_git_depth.try_into().unwrap_or(0), + ) }) .flatten() .collect::>(); diff --git a/src/priority.rs b/src/priority.rs index f928e74..279ce62 100644 --- a/src/priority.rs +++ b/src/priority.rs @@ -1,4 +1,4 @@ -use git2::Repository; +use git2; use regex; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, path::Path}; @@ -61,7 +61,11 @@ pub fn compute_recentness_boost( /// Get the commit time of the most recent change to each file using git2. /// Returns a map from file path (relative to the repo root) → last commit Unix time. /// If Git or .git folder is missing, returns None instead of erroring. -pub fn get_recent_commit_times_git2(repo_path: &Path) -> Option> { +/// Only considers up to `max_commits` most recent commits. +pub fn get_recent_commit_times_git2( + repo_path: &Path, + max_commits: usize, +) -> Option> { // Walk up until you find a .git folder but not higher than the base of the given repo_path let mut current_path = repo_path.to_path_buf(); while current_path.components().count() > 1 { @@ -71,7 +75,7 @@ pub fn get_recent_commit_times_git2(repo_path: &Path) -> Option repo, Err(_) => { debug!("Not a Git repository or unable to open: {:?}", current_path); @@ -97,14 +101,15 @@ pub fn get_recent_commit_times_git2(repo_path: &Path) -> Option oid, Err(e) => { debug!("Error during revwalk iteration: {:?}", e); continue; } }; + let commit = match repo.find_commit(oid) { Ok(commit) => commit, Err(e) => { @@ -119,8 +124,8 @@ pub fn get_recent_commit_times_git2(repo_path: &Path) -> Option