Use gitignore in streaming mode as well

.github/workflows/release.yml

@@ -3,13 +3,17 @@ on:
   push:
     tags:
       - "v*"
+    branches:
+      - main
+  workflow_dispatch:
 
 permissions:
   contents: write
   packages: write
 
 jobs:
   build:
+    if: github.ref_name == 'main' && startsWith(github.ref, 'refs/tags/v')
     strategy:
       matrix:
         include:
@@ -84,6 +88,7 @@ jobs:
           path: sha-${{ matrix.target }}.txt
 
   release:
+    if: github.ref_name == 'main' && startsWith(github.ref, 'refs/tags/v')
     needs: build
     runs-on: ubuntu-latest
     steps:
@@ -105,8 +110,10 @@ jobs:
           generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
 
   publish:
+    if: github.ref_name == 'main' && startsWith(github.ref, 'refs/tags/v')
     name: Publish to crates.io
     needs: [release]
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.13.8] - 2025-01-20
+[0.13.8]: https://github.com/bodo-run/yek/compare/v0.13.7...v0.13.8
+### Bug Fixes
+
+- Use WalkBuilder in streaming mode to respect gitignore
+- Include hidden files in WalkBuilder configuration
+
+### Miscellaneous Tasks
+
+- Fix the release script
+
+### Refactor
+
+- Improve gitignore handling and fix clippy warnings
+- Improve binary file handling and remove duplicate gitignore checks
+
+### Testing
+
+- Add comprehensive gitignore end-to-end tests
+- Fix binary file test assertion
+
+### Ci
+
+- Simpler release script
+
 ## [0.13.7] - 2025-01-19
 [0.13.7]: https://github.com/bodo-run/yek/compare/v0.13.5...v0.13.7
 ### Bug Fixes

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "yek"
-version = "0.13.7"
+version = "0.13.8"
 edition = "2021"
 
 [dependencies]

scripts/make-release.sh

@@ -45,14 +45,14 @@ echo "Bumping version to: $NEW_VERSION"
 
 # 4. Generate/Update CHANGELOG using cargo-cliff
 #    Make sure cargo-cliff is installed (cargo install cargo-cliff)
-cargo cliff --tag "v${NEW_VERSION}" --output CHANGELOG.md
+git cliff --tag "v${NEW_VERSION}" --output CHANGELOG.md
 
 # 5. Update Cargo.toml
 sed -i.bak "s/^version *= *\"${CURRENT_VERSION}\"/version = \"${NEW_VERSION}\"/" Cargo.toml
 rm -f Cargo.toml.bak
 
 # 6. Update Cargo.lock (so that if your package references itself, it's updated)
-cargo update -p "$(cargo pkgid | sed 's|.*#||')"
+cargo update -p yek
 
 # 7. Commit changes
 git add Cargo.toml Cargo.lock CHANGELOG.md

src/lib.rs

@@ -1,5 +1,6 @@
 use anyhow::Result;
 use ignore::gitignore::GitignoreBuilder;
+use ignore::WalkBuilder;
 use regex::Regex;
 use serde::Deserialize;
 use std::collections::HashMap;
@@ -8,7 +9,6 @@ use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 use std::process::{Command as SysCommand, Stdio};
 use tracing::debug;
-use walkdir::WalkDir;
 mod parallel;
 use parallel::process_files_parallel;
 use path_slash::PathExt;
@@ -562,7 +562,7 @@ pub fn serialize_repo(
     if gitignore_path.exists() {
         builder.add(&gitignore_path);
     }
-    let gitignore = builder
+    let _gitignore = builder
         .build()
         .unwrap_or_else(|_| GitignoreBuilder::new(base_path).build().unwrap());
 
@@ -595,67 +595,73 @@ pub fn serialize_repo(
         // 1) Collect all FileEntry objects
         let mut files: Vec<FileEntry> = Vec::new();
 
-        for entry in WalkDir::new(base_path)
-            .follow_links(true)
-            .into_iter()
-            .filter_map(|e| e.ok())
-        {
-            let path = entry.path();
-            if !path.is_file() {
-                continue;
-            }
-
-            // Get path relative to base
-            let rel_str = normalize_path(base_path, path);
-
-            // Skip via .gitignore
-            if gitignore
-                .matched(path.strip_prefix(base_path).unwrap_or(path), false)
-                .is_ignore()
-            {
-                debug!("Skipping {} - matched by gitignore", rel_str);
-                continue;
-            }
+        let mut builder = WalkBuilder::new(base_path);
+        builder
+            .follow_links(false)
+            .standard_filters(true)
+            .hidden(false)
+            .git_ignore(true)
+            .git_global(false)
+            .git_exclude(false)
+            .require_git(false);
+
+        for entry in builder.build().flatten() {
+            if entry.file_type().is_some_and(|ft| ft.is_file()) {
+                let path = entry.path();
+                let rel_str = normalize_path(base_path, path);
+
+                // Skip via our ignore regexes
+                if final_config
+                    .ignore_patterns
+                    .iter()
+                    .any(|pat| pat.is_match(&rel_str))
+                {
+                    debug!("Skipping {} - matched ignore pattern", rel_str);
+                    continue;
+                }
 
-            // Skip via our ignore regexes
-            if final_config
-                .ignore_patterns
-                .iter()
-                .any(|pat| pat.is_match(&rel_str))
-            {
-                debug!("Skipping {} - matched ignore pattern", rel_str);
-                continue;
-            }
+                // Skip .gitignore files
+                if path.file_name().is_some_and(|f| f == ".gitignore") {
+                    debug!("Skipping .gitignore file");
+                    continue;
+                }
 
-            // Check if text or binary
-            let user_bin_exts = config
-                .as_ref()
-                .map(|c| c.binary_extensions.as_slice())
-                .unwrap_or(&[]);
-            if !is_text_file(path, user_bin_exts) {
-                debug!("Skipping binary file: {}", rel_str);
-                continue;
-            }
+                // Skip via gitignore
+                if entry.path().file_name().is_some_and(|f| f == ".gitignore") {
+                    debug!("Skipping .gitignore file");
+                    continue;
+                }
 
-            // Calculate priority with recentness boost
-            let mut priority = get_file_priority(
-                &rel_str,
-                &final_config.ignore_patterns,
-                &final_config.priority_list,
-            );
+                // Skip binary files
+                if !is_text_file(
+                    path,
+                    config
+                        .as_ref()
+                        .map(|c| &c.binary_extensions)
+                        .unwrap_or(&vec![]),
+                ) {
+                    debug!("Skipping binary file: {}", rel_str);
+                    continue;
+                }
 
-            // Apply recentness boost if available
-            if let Some(boost_map) = recentness_boost.as_ref() {
-                if let Some(boost) = boost_map.get(&rel_str) {
-                    priority += *boost;
+                // Apply recentness boost to priority
+                let mut priority = get_file_priority(
+                    &rel_str,
+                    &final_config.ignore_patterns,
+                    &final_config.priority_list,
+                );
+                if let Some(boost_map) = &recentness_boost {
+                    if let Some(boost) = boost_map.get(&rel_str) {
+                        priority += boost;
+                    }
                 }
-            }
 
-            files.push(FileEntry {
-                path: path.to_path_buf(),
-                priority,
-                file_index: files.len(),
-            });
+                files.push(FileEntry {
+                    path: path.to_path_buf(),
+                    priority,
+                    file_index: files.len(),
+                });
+            }
         }
 
         // 2) Sort ascending by priority, so the last entries are the most important

src/parallel.rs

@@ -3,7 +3,7 @@ use crate::{
     PriorityPattern, Result, YekConfig,
 };
 use crossbeam::channel::{bounded, Receiver, Sender};
-use ignore::{gitignore::GitignoreBuilder, WalkBuilder};
+use ignore::WalkBuilder;
 use num_cpus::get;
 use regex::Regex;
 use std::{
@@ -218,56 +218,48 @@ fn collect_files(
     priority_list: &[PriorityPattern],
     recentness_boost: Option<&HashMap<String, i32>>,
 ) -> Result<Vec<FileEntry>> {
-    // Build gitignore matcher
-    let mut builder = GitignoreBuilder::new(base_dir);
-    let gitignore_path = base_dir.join(".gitignore");
-    if gitignore_path.exists() {
-        builder.add(&gitignore_path);
-    }
-    let gitignore = builder
-        .build()
-        .unwrap_or_else(|_| GitignoreBuilder::new(base_dir).build().unwrap());
-
     let mut builder = WalkBuilder::new(base_dir);
     builder
         .follow_links(false)
         .standard_filters(true)
-        .add_custom_ignore_filename(".gitignore")
+        .hidden(false)
+        .git_ignore(true)
+        .git_global(false)
+        .git_exclude(false)
         .require_git(false);
 
     let mut results = Vec::new();
     let mut file_index = 0;
 
     for entry in builder.build().flatten() {
         if entry.file_type().is_some_and(|ft| ft.is_file()) {
-            let path = entry.path().to_path_buf();
-            let rel_str = normalize_path(base_dir, &path);
-            let rel_path = path.strip_prefix(base_dir).unwrap_or(&path);
-
-            // Skip via .gitignore
-            if gitignore.matched(rel_path, false).is_ignore() {
-                debug!("Skipping {} - matched by gitignore", rel_str);
-                continue;
-            }
+            let path = entry.path();
+            let rel_str = normalize_path(base_dir, path);
 
             // Skip via our ignore regexes
             if ignore_patterns.iter().any(|p| p.is_match(&rel_str)) {
                 debug!("Skipping {} - matched ignore pattern", rel_str);
                 continue;
             }
 
-            // Check if text or binary
+            // Skip .gitignore files
+            if path.file_name().is_some_and(|f| f == ".gitignore") {
+                debug!("Skipping .gitignore file");
+                continue;
+            }
+
+            // Skip binary files
             let user_bin_exts = config
                 .as_ref()
                 .map(|c| c.binary_extensions.as_slice())
                 .unwrap_or(&[]);
-            if !is_text_file(&path, user_bin_exts) {
+            if !is_text_file(path, user_bin_exts) {
                 debug!("Skipping binary file: {}", rel_str);
                 continue;
             }
 
             results.push(FileEntry {
-                path,
+                path: path.to_path_buf(),
                 priority: get_file_priority(&rel_str, ignore_patterns, priority_list),
                 file_index,
             });