From 4e5cbce5126fdd93ca5a3c99b9d96ef1b18a5a42 Mon Sep 17 00:00:00 2001 From: shu-kitamura Date: Thu, 12 Feb 2026 21:53:02 +0900 Subject: [PATCH 1/8] Add copilot instruction for pr review --- .../review/pr-review.instructions.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/instructions/review/pr-review.instructions.md diff --git a/.github/instructions/review/pr-review.instructions.md b/.github/instructions/review/pr-review.instructions.md new file mode 100644 index 0000000..c9c7c1f --- /dev/null +++ b/.github/instructions/review/pr-review.instructions.md @@ -0,0 +1,24 @@ +# GitHub Copilot Instructions + +You are a senior code reviewer ensuring high standards of code quality + +When performing a code review, **respond in Japanese**. +Please focus on the following checklist: + +## Checklist + +- Code is simple and readable +- Functions and variables are well-named +- No duplicated code +- Proper error handling +- No exposed secrets or API keys +- Input validation implemented +- Good test coverage +- Performance considerations addressed +- Time complexity of algorithms analyzed +- Licenses of integrated libraries checked +- Large functions (>50 lines) +- Large files (>800 lines) +- Deep nesting (>4 levels) +- Hardcoded configuration values +- Magic numbers without explanation From 0498dcaf35384d3a1760ce6b5233e0da02797013 Mon Sep 17 00:00:00 2001 From: shu-kitamura Date: Thu, 12 Feb 2026 21:57:41 +0900 Subject: [PATCH 2/8] Update .github/instructions/review/pr-review.instructions.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/instructions/review/pr-review.instructions.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/instructions/review/pr-review.instructions.md b/.github/instructions/review/pr-review.instructions.md index c9c7c1f..472e686 100644 --- a/.github/instructions/review/pr-review.instructions.md +++ b/.github/instructions/review/pr-review.instructions.md @@ -17,8 +17,8 @@ Please focus on the following checklist: - Performance considerations addressed - Time complexity of algorithms analyzed - Licenses of integrated libraries checked -- Large functions (>50 lines) -- Large files (>800 lines) -- Deep nesting (>4 levels) -- Hardcoded configuration values -- Magic numbers without explanation +- No overly large functions (>50 lines) +- No overly large files (>800 lines) +- No deeply nested logic (>4 levels) +- No hardcoded configuration values; configuration is externalized +- No unexplained magic numbers; constants are named and documented From 9ee8c7472d175e4bbd53fb79906d321699d2f06e Mon Sep 17 00:00:00 2001 From: shu-kitamura Date: Thu, 12 Feb 2026 21:57:50 +0900 Subject: [PATCH 3/8] Update .github/instructions/review/pr-review.instructions.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/instructions/review/pr-review.instructions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/instructions/review/pr-review.instructions.md b/.github/instructions/review/pr-review.instructions.md index 472e686..cc51800 100644 --- a/.github/instructions/review/pr-review.instructions.md +++ b/.github/instructions/review/pr-review.instructions.md @@ -16,7 +16,7 @@ Please focus on the following checklist: - Good test coverage - Performance considerations addressed - Time complexity of algorithms analyzed -- Licenses of integrated libraries checked +- Integrated libraries have compatible licenses (no license conflicts in dependencies) - No overly large functions (>50 lines) - No overly large files (>800 lines) - No deeply nested logic (>4 levels) From 0ed2fba462b21e92255b84b318d94ff67e96ef6b Mon Sep 17 00:00:00 2001 From: shu-kitamura Date: Thu, 12 Feb 2026 22:39:49 +0900 Subject: [PATCH 4/8] feat: extract must literal substrings from AST --- crates/regex-core/src/engine.rs | 4 +- crates/regex-core/src/engine/ast.rs | 192 ++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 1 deletion(-) diff --git a/crates/regex-core/src/engine.rs b/crates/regex-core/src/engine.rs index edbb4f5..e14f11a 100644 --- a/crates/regex-core/src/engine.rs +++ b/crates/regex-core/src/engine.rs @@ -13,6 +13,7 @@ use crate::engine::{ parser::parse, }; +pub(crate) use ast::{Ast, extract_must_literals}; pub use compiler::CompileError; pub use evaluator::EvalError; pub use instruction::Instruction; @@ -62,7 +63,8 @@ where /// Parse and compile a pattern. pub fn compile_pattern(pattern: &str) -> Result, RegexError> { - let ast = parse(pattern)?; + let ast: Ast = parse(pattern)?; + let _must_literals = extract_must_literals(&ast); let instructions = compile(&ast)?; Ok(instructions) } diff --git a/crates/regex-core/src/engine/ast.rs b/crates/regex-core/src/engine/ast.rs index 81ef826..047d925 100644 --- a/crates/regex-core/src/engine/ast.rs +++ b/crates/regex-core/src/engine/ast.rs @@ -1,5 +1,10 @@ //! AST definitions for the regex engine. +use std::{cmp::Ordering, collections::BTreeSet}; + +/// Maximum number of must literals to retain. +pub(crate) const MUST_LITERAL_LIMIT: usize = 16; + /// Inclusive character range. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct CharRange { @@ -111,3 +116,190 @@ pub enum Ast { /// Backreference node (`\1`, `\2`, ...). Backreference(usize), } + +/// Extracts conservative must substrings from an AST. +/// +/// Each returned string is guaranteed to appear in every successful match. +/// The result is trimmed to at most `MUST_LITERAL_LIMIT` entries by: +/// 1. longer byte length first +/// 2. lexicographic byte order for equal lengths +pub(crate) fn extract_must_literals(ast: &Ast) -> Vec { + let mut must = extract_must_literal_set(ast); + prune_must_literal_set(&mut must); + must_literal_set_to_vec(must) +} + +fn extract_must_literal_set(ast: &Ast) -> BTreeSet { + match ast { + Ast::Empty | Ast::Assertion(_) | Ast::Backreference(_) => BTreeSet::new(), + Ast::CharClass(class) => { + let mut must = BTreeSet::new(); + if let Some(literal) = class_single_literal(class) { + must.insert(literal.to_string()); + } + must + } + Ast::Capture { expr, .. } => extract_must_literal_set(expr), + Ast::ZeroOrMore { .. } | Ast::ZeroOrOne { .. } => BTreeSet::new(), + Ast::OneOrMore { expr, .. } => extract_must_literal_set(expr), + Ast::Repeat { expr, min, .. } => { + if *min == 0 { + BTreeSet::new() + } else { + extract_must_literal_set(expr) + } + } + Ast::Concat(exprs) => extract_concat_must_literals(exprs), + Ast::Alternate(left, right) => { + let left_set = extract_must_literal_set(left); + let right_set = extract_must_literal_set(right); + intersect_must_literal_sets(left_set, right_set) + } + } +} + +fn extract_concat_must_literals(exprs: &[Ast]) -> BTreeSet { + let mut must = BTreeSet::new(); + let mut literal_run = String::new(); + + for expr in exprs { + if let Some(literal) = ast_single_literal(expr) { + literal_run.push_str(&literal); + continue; + } + + flush_literal_run(&mut must, &mut literal_run); + union_must_literal_sets(&mut must, extract_must_literal_set(expr)); + } + + flush_literal_run(&mut must, &mut literal_run); + prune_must_literal_set(&mut must); + must +} + +fn ast_single_literal(ast: &Ast) -> Option { + let Ast::CharClass(class) = ast else { + return None; + }; + class_single_literal(class).map(|c| c.to_string()) +} + +fn class_single_literal(class: &CharClass) -> Option { + if class.negated || class.ranges.len() != 1 { + return None; + } + + let range = class.ranges.first()?; + if range.start == range.end { + Some(range.start) + } else { + None + } +} + +fn flush_literal_run(set: &mut BTreeSet, literal_run: &mut String) { + if literal_run.is_empty() { + return; + } + + set.insert(std::mem::take(literal_run)); + prune_must_literal_set(set); +} + +fn union_must_literal_sets(dst: &mut BTreeSet, src: BTreeSet) { + dst.extend(src); + prune_must_literal_set(dst); +} + +fn intersect_must_literal_sets( + left: BTreeSet, + right: BTreeSet, +) -> BTreeSet { + let mut intersection = BTreeSet::new(); + for literal in left { + if right.contains(&literal) { + intersection.insert(literal); + } + } + prune_must_literal_set(&mut intersection); + intersection +} + +fn prune_must_literal_set(set: &mut BTreeSet) { + if set.len() <= MUST_LITERAL_LIMIT { + return; + } + + let mut literals: Vec = set.iter().cloned().collect(); + literals.sort_by(|a, b| compare_literals(a, b)); + literals.truncate(MUST_LITERAL_LIMIT); + *set = literals.into_iter().collect(); +} + +fn must_literal_set_to_vec(set: BTreeSet) -> Vec { + let mut literals: Vec = set.into_iter().collect(); + literals.sort_by(|a, b| compare_literals(a, b)); + literals.truncate(MUST_LITERAL_LIMIT); + literals +} + +fn compare_literals(a: &str, b: &str) -> Ordering { + b.len() + .cmp(&a.len()) + .then_with(|| a.as_bytes().cmp(b.as_bytes())) +} + +#[cfg(test)] +mod tests { + use super::{MUST_LITERAL_LIMIT, extract_must_literals}; + use crate::engine::parser::parse; + + #[test] + fn test_extract_must_literals_dot_star_abc_dot_star() { + let ast = parse(".*abc.*").unwrap(); + let actual = extract_must_literals(&ast); + assert_eq!(actual, vec!["abc".to_string()]); + } + + #[test] + fn test_extract_must_literals_alternate_no_common_literal() { + let ast = parse("(abc|def)").unwrap(); + let actual = extract_must_literals(&ast); + assert_eq!(actual, Vec::::new()); + } + + #[test] + fn test_extract_must_literals_ab_star_c() { + let ast = parse("ab*c").unwrap(); + let actual = extract_must_literals(&ast); + assert_eq!(actual, vec!["a".to_string(), "c".to_string()]); + } + + #[test] + fn test_extract_must_literals_a_class_z() { + let ast = parse("a[a-z]z").unwrap(); + let actual = extract_must_literals(&ast); + assert_eq!(actual, vec!["a".to_string(), "z".to_string()]); + } + + #[test] + fn test_extract_must_literals_limit_prefers_longer_then_lexicographic() { + let pattern = [ + "ppp", "aaa", "qqq", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii", "jjj", + "kkk", "lll", "mmm", "nnn", "ooo", "zzzz", + ] + .join(".*"); + let ast = parse(&pattern).unwrap(); + + let actual = extract_must_literals(&ast); + assert_eq!(actual.len(), MUST_LITERAL_LIMIT); + let expected = [ + "zzzz", "aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii", "jjj", "kkk", + "lll", "mmm", "nnn", "ooo", + ] + .into_iter() + .map(str::to_string) + .collect::>(); + assert_eq!(actual, expected); + } +} From 61fdbc4c6a1202d6e0bd54748e6ec5895319a38a Mon Sep 17 00:00:00 2001 From: shu-kitamura Date: Fri, 13 Feb 2026 23:01:03 +0900 Subject: [PATCH 5/8] feat: use must literals in match --- crates/regex-core/src/engine.rs | 38 +++--- crates/regex-core/src/engine/evaluator.rs | 19 +-- crates/regex-core/src/lib.rs | 149 +++++++--------------- 3 files changed, 63 insertions(+), 143 deletions(-) diff --git a/crates/regex-core/src/engine.rs b/crates/regex-core/src/engine.rs index e14f11a..5f0909d 100644 --- a/crates/regex-core/src/engine.rs +++ b/crates/regex-core/src/engine.rs @@ -7,11 +7,7 @@ mod parser; use thiserror::Error; -use crate::engine::{ - compiler::compile, - evaluator::{eval, eval_from_start}, - parser::parse, -}; +use crate::engine::{compiler::compile, evaluator::eval, parser::parse}; pub(crate) use ast::{Ast, extract_must_literals}; pub use compiler::CompileError; @@ -61,12 +57,14 @@ where } } -/// Parse and compile a pattern. -pub fn compile_pattern(pattern: &str) -> Result, RegexError> { +/// Parse, extract must literals, and compile a pattern. +pub(crate) fn compile_pattern_with_must_literals( + pattern: &str, +) -> Result<(Vec, Vec), RegexError> { let ast: Ast = parse(pattern)?; - let _must_literals = extract_must_literals(&ast); + let must_literals = extract_must_literals(&ast); let instructions = compile(&ast)?; - Ok(instructions) + Ok((instructions, must_literals)) } /// Match an instruction sequence against a line. @@ -74,28 +72,23 @@ pub fn match_line(code: &[Instruction], line: &str) -> Result Ok(eval(code, line)?) } -/// Match an instruction sequence from the start of a line. -pub fn match_line_from_start(code: &[Instruction], line: &str) -> Result { - Ok(eval_from_start(code, line)?) -} - #[cfg(test)] mod tests { use crate::engine::{ - CompileError, RegexError, compile_pattern, instruction::Instruction, match_line, - match_line_from_start, + CompileError, RegexError, compile_pattern_with_must_literals, instruction::Instruction, + match_line, }; #[test] fn test_compile_pattern_literal() { - let code = compile_pattern("abc").unwrap(); + let (code, _) = compile_pattern_with_must_literals("abc").unwrap(); assert_eq!(code.len(), 4); assert!(matches!(code.last(), Some(Instruction::Match))); } #[test] fn test_compile_pattern_invalid_backreference() { - let actual = compile_pattern("(a)\\2"); + let actual = compile_pattern_with_must_literals("(a)\\2"); assert_eq!( actual, Err(RegexError::Compile(CompileError::InvalidBackreference(2))) @@ -104,15 +97,14 @@ mod tests { #[test] fn test_match_line_backreference() { - let code = compile_pattern("(abc)\\1").unwrap(); + let (code, _) = compile_pattern_with_must_literals("(abc)\\1").unwrap(); assert!(match_line(&code, "abcabc").unwrap()); assert!(!match_line(&code, "abcabd").unwrap()); } #[test] - fn test_match_line_from_start() { - let code = compile_pattern("abc").unwrap(); - assert!(match_line_from_start(&code, "abcdef").unwrap()); - assert!(!match_line_from_start(&code, "zabc").unwrap()); + fn test_compile_pattern_with_must_literals() { + let (_code, must_literals) = compile_pattern_with_must_literals(".*abc.*").unwrap(); + assert_eq!(must_literals, vec!["abc".to_string()]); } } diff --git a/crates/regex-core/src/engine/evaluator.rs b/crates/regex-core/src/engine/evaluator.rs index 6b24f0d..8e8a213 100644 --- a/crates/regex-core/src/engine/evaluator.rs +++ b/crates/regex-core/src/engine/evaluator.rs @@ -251,15 +251,6 @@ fn eval_from_start_inner( Ok(false) } -/// Evaluates whether `input` matches from the first character. -pub fn eval_from_start(inst: &[Instruction], input: &str) -> Result { - let chars: Vec = input.chars().collect(); - let capture_slots = max_capture_index(inst) - .checked_add(1) - .ok_or(EvalError::PCOverFlow)?; - eval_from_start_inner(inst, &chars, 0, capture_slots) -} - /// Evaluates whether `input` matches at any starting position. pub fn eval(inst: &[Instruction], input: &str) -> Result { let chars: Vec = input.chars().collect(); @@ -281,7 +272,7 @@ mod tests { use crate::engine::{ ast::{CharClass, CharRange, Predicate}, compiler::compile, - evaluator::{EvalError, eval, eval_from_start}, + evaluator::{EvalError, eval}, instruction::Instruction, parser::parse, }; @@ -350,12 +341,4 @@ mod tests { let actual = eval(&inst, "abc"); assert_eq!(actual, Err(EvalError::InvalidPC)); } - - #[test] - fn test_eval_from_start() { - let ast = parse("abc").unwrap(); - let inst = compile(&ast).unwrap(); - assert!(eval_from_start(&inst, "abcxxx").unwrap()); - assert!(!eval_from_start(&inst, "xabc").unwrap()); - } } diff --git a/crates/regex-core/src/lib.rs b/crates/regex-core/src/lib.rs index 3f1fa22..fd24521 100644 --- a/crates/regex-core/src/lib.rs +++ b/crates/regex-core/src/lib.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeSet; - use engine::Instruction; mod engine; @@ -9,8 +7,8 @@ pub mod error; pub struct Regex { /// Compiled instruction sequence. code: Vec, - /// Literal prefixes used for a fast pre-filter. - first_strings: BTreeSet, + /// Must-have literal substrings used for a fast pre-filter. + must_literals: Vec, /// Enables case-insensitive matching by lowercasing pattern/input. is_ignore_case: bool, /// Inverts the final match result. @@ -24,17 +22,15 @@ impl Regex { is_ignore_case: bool, is_invert_match: bool, ) -> Result { - let code = if is_ignore_case { - engine::compile_pattern(&pattern.to_lowercase())? + let (code, must_literals) = if is_ignore_case { + engine::compile_pattern_with_must_literals(&pattern.to_lowercase())? } else { - engine::compile_pattern(pattern)? + engine::compile_pattern_with_must_literals(pattern)? }; - let first_strings = Self::get_first_strings(&code); - Ok(Self { code, - first_strings, + must_literals, is_ignore_case, is_invert_match, }) @@ -51,95 +47,20 @@ impl Regex { Ok(is_match ^ self.is_invert_match) } - /// Matches a line, optionally using a prefix pre-filter first. + /// Matches a line, optionally using a must-literal pre-filter first. fn is_match_line(&self, line: &str) -> Result { - if self.first_strings.is_empty() { - return engine::match_line(&self.code, line); - } - - let mut pos = 0; - while let Some(i) = find_index(&line[pos..], &self.first_strings) { - let start = pos + i; - if engine::match_line_from_start(&self.code, &line[start..])? { - return Ok(true); - } - pos = start + 1; - } - - Ok(false) - } - - /// Extracts deterministic literal prefixes from the instruction stream. - fn get_first_strings(insts: &[Instruction]) -> BTreeSet { - let mut first_strings: BTreeSet = BTreeSet::new(); - match insts.first() { - Some(inst) if Self::literal_from_instruction(inst).is_some() => { - if let Some(string) = Self::get_string(insts, 0) { - first_strings.insert(string); - } - } - Some(Instruction::Split(left, right)) => { - if let Some(string) = Self::get_string(insts, *left) { - first_strings.insert(string); - } - if let Some(string) = Self::get_string(insts, *right) { - first_strings.insert(string); - } - } - _ => {} - } - first_strings - } - - /// Collects a run of literal characters starting at `start`. - fn get_string(insts: &[Instruction], mut start: usize) -> Option { - let mut pre: String = String::new(); - - while start < insts.len() { - let Some(inst) = insts.get(start) else { - break; - }; - - match Self::literal_from_instruction(inst) { - Some(c) => { - pre.push(c); - start += 1; - } - None => break, - } - } - - if pre.is_empty() { None } else { Some(pre) } - } - - /// Returns a literal character when the instruction is a single-char class. - fn literal_from_instruction(inst: &Instruction) -> Option { - let Instruction::CharClass(class) = inst else { - return None; - }; - - if class.negated || class.ranges.len() != 1 { - return None; + if !self + .must_literals + .iter() + .all(|literal| line.contains(literal)) + { + return Ok(false); } - let range = class.ranges.first()?; - if range.start == range.end { - Some(range.start) - } else { - None - } + engine::match_line(&self.code, line) } } -/// Returns the smallest found index among all candidate strings. -fn find_index(string: &str, string_set: &BTreeSet) -> Option { - string_set - .iter() - .map(|s| string.find(s)) - .filter(|opt| opt.is_some()) - .min()? -} - #[cfg(test)] mod tests { use super::*; @@ -200,14 +121,38 @@ mod tests { } #[test] - fn test_get_first_strings() { - let regex = Regex::new("abc", false, false).unwrap(); - assert_eq!(regex.first_strings.len(), 1); - assert!(regex.first_strings.contains("abc")); - - let regex = Regex::new("a*bc", false, false).unwrap(); - assert_eq!(regex.first_strings.len(), 2); - assert!(regex.first_strings.contains("a")); - assert!(regex.first_strings.contains("bc")); + fn test_extracts_must_literals_for_filtering() { + let regex = Regex::new(".*abc.*", false, false).unwrap(); + assert_eq!(regex.must_literals, vec!["abc".to_string()]); + + let regex = Regex::new("ab*c", false, false).unwrap(); + assert_eq!(regex.must_literals, vec!["a".to_string(), "c".to_string()]); + } + + #[test] + fn test_must_literal_filter_skips_non_matching_lines() { + let regex = Regex::new(".*abc.*", false, false).unwrap(); + assert!(!regex.is_match("zzz").unwrap()); + } + + #[test] + fn test_must_literal_filter_allows_matching_lines() { + let regex = Regex::new("a.*c", false, false).unwrap(); + assert!(regex.is_match("a---c").unwrap()); + assert!(!regex.is_match("a---").unwrap()); + } + + #[test] + fn test_must_literal_filter_respects_invert_match() { + let regex = Regex::new(".*abc.*", false, true).unwrap(); + assert!(regex.is_match("zzz").unwrap()); + } + + #[test] + fn test_empty_must_literals_still_runs_matcher() { + let regex = Regex::new("(abc|def)", false, false).unwrap(); + assert!(regex.must_literals.is_empty()); + assert!(regex.is_match("def").unwrap()); + assert!(!regex.is_match("xyz").unwrap()); } } From 16f7a8cc8b088bbf6eba9ae05c0872e07ec81466 Mon Sep 17 00:00:00 2001 From: shu-kitamura Date: Fri, 13 Feb 2026 23:40:41 +0900 Subject: [PATCH 6/8] feat: add needles and nullable for ast --- crates/regex-core/src/engine.rs | 6 +- crates/regex-core/src/engine/ast.rs | 241 ++++++++++++++++++++++------ 2 files changed, 198 insertions(+), 49 deletions(-) diff --git a/crates/regex-core/src/engine.rs b/crates/regex-core/src/engine.rs index 5f0909d..a973aed 100644 --- a/crates/regex-core/src/engine.rs +++ b/crates/regex-core/src/engine.rs @@ -9,7 +9,7 @@ use thiserror::Error; use crate::engine::{compiler::compile, evaluator::eval, parser::parse}; -pub(crate) use ast::{Ast, extract_must_literals}; +pub(crate) use ast::{Ast, analyze_ast}; pub use compiler::CompileError; pub use evaluator::EvalError; pub use instruction::Instruction; @@ -62,9 +62,9 @@ pub(crate) fn compile_pattern_with_must_literals( pattern: &str, ) -> Result<(Vec, Vec), RegexError> { let ast: Ast = parse(pattern)?; - let must_literals = extract_must_literals(&ast); + let analysis = analyze_ast(&ast); let instructions = compile(&ast)?; - Ok((instructions, must_literals)) + Ok((instructions, analysis.must_literals)) } /// Match an instruction sequence against a line. diff --git a/crates/regex-core/src/engine/ast.rs b/crates/regex-core/src/engine/ast.rs index 047d925..01a97e7 100644 --- a/crates/regex-core/src/engine/ast.rs +++ b/crates/regex-core/src/engine/ast.rs @@ -117,64 +117,150 @@ pub enum Ast { Backreference(usize), } +/// Aggregate analysis results derived from one AST. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct AstAnalysis { + /// Conservative must substrings that appear in every successful match. + pub must_literals: Vec, + /// Candidate literal substrings used as prefilter hints. + pub needles: Vec, + /// Whether this pattern can match the empty string. + pub nullable: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AstAnalysisSet { + must_literals: BTreeSet, + needles: BTreeSet, + nullable: bool, +} + +/// Analyzes `ast` and returns deterministic `must_literals`, `needles`, and `nullable`. +pub(crate) fn analyze_ast(ast: &Ast) -> AstAnalysis { + let result = analyze_ast_set(ast); + AstAnalysis { + must_literals: literal_set_to_vec(result.must_literals), + needles: literal_set_to_vec(result.needles), + nullable: result.nullable, + } +} + /// Extracts conservative must substrings from an AST. /// /// Each returned string is guaranteed to appear in every successful match. -/// The result is trimmed to at most `MUST_LITERAL_LIMIT` entries by: -/// 1. longer byte length first -/// 2. lexicographic byte order for equal lengths +#[allow(dead_code)] pub(crate) fn extract_must_literals(ast: &Ast) -> Vec { - let mut must = extract_must_literal_set(ast); - prune_must_literal_set(&mut must); - must_literal_set_to_vec(must) + analyze_ast(ast).must_literals +} + +/// Extracts candidate literal substrings from an AST. +#[allow(dead_code)] +pub(crate) fn extract_needles(ast: &Ast) -> Vec { + analyze_ast(ast).needles +} + +/// Returns whether the pattern represented by `ast` can match an empty string. +#[allow(dead_code)] +pub(crate) fn is_nullable(ast: &Ast) -> bool { + analyze_ast(ast).nullable } -fn extract_must_literal_set(ast: &Ast) -> BTreeSet { +fn analyze_ast_set(ast: &Ast) -> AstAnalysisSet { match ast { - Ast::Empty | Ast::Assertion(_) | Ast::Backreference(_) => BTreeSet::new(), - Ast::CharClass(class) => { - let mut must = BTreeSet::new(); - if let Some(literal) = class_single_literal(class) { - must.insert(literal.to_string()); + Ast::Empty | Ast::Assertion(_) => AstAnalysisSet { + must_literals: BTreeSet::new(), + needles: BTreeSet::new(), + nullable: true, + }, + Ast::Backreference(_) => AstAnalysisSet { + must_literals: BTreeSet::new(), + needles: BTreeSet::new(), + nullable: false, + }, + Ast::CharClass(class) => analyze_char_class(class), + Ast::Capture { expr, .. } => analyze_ast_set(expr), + Ast::ZeroOrMore { expr, .. } | Ast::ZeroOrOne { expr, .. } => { + let child = analyze_ast_set(expr); + AstAnalysisSet { + must_literals: BTreeSet::new(), + needles: child.needles, + nullable: true, } - must } - Ast::Capture { expr, .. } => extract_must_literal_set(expr), - Ast::ZeroOrMore { .. } | Ast::ZeroOrOne { .. } => BTreeSet::new(), - Ast::OneOrMore { expr, .. } => extract_must_literal_set(expr), + Ast::OneOrMore { expr, .. } => analyze_ast_set(expr), Ast::Repeat { expr, min, .. } => { - if *min == 0 { - BTreeSet::new() - } else { - extract_must_literal_set(expr) + let child = analyze_ast_set(expr); + AstAnalysisSet { + must_literals: if *min == 0 { + BTreeSet::new() + } else { + child.must_literals + }, + needles: child.needles, + nullable: if *min == 0 { true } else { child.nullable }, } } - Ast::Concat(exprs) => extract_concat_must_literals(exprs), - Ast::Alternate(left, right) => { - let left_set = extract_must_literal_set(left); - let right_set = extract_must_literal_set(right); - intersect_must_literal_sets(left_set, right_set) - } + Ast::Concat(exprs) => analyze_concat(exprs), + Ast::Alternate(left, right) => analyze_alternate(left, right), + } +} + +fn analyze_char_class(class: &CharClass) -> AstAnalysisSet { + let mut must_literals = BTreeSet::new(); + let mut needles = BTreeSet::new(); + if let Some(literal) = class_single_literal(class) { + let literal = literal.to_string(); + must_literals.insert(literal.clone()); + needles.insert(literal); + } + AstAnalysisSet { + must_literals, + needles, + nullable: false, } } -fn extract_concat_must_literals(exprs: &[Ast]) -> BTreeSet { - let mut must = BTreeSet::new(); +fn analyze_concat(exprs: &[Ast]) -> AstAnalysisSet { + let mut must_literals = BTreeSet::new(); + let mut needles = BTreeSet::new(); let mut literal_run = String::new(); + let mut nullable = true; for expr in exprs { + let child = analyze_ast_set(expr); + nullable &= child.nullable; + if let Some(literal) = ast_single_literal(expr) { literal_run.push_str(&literal); continue; } - flush_literal_run(&mut must, &mut literal_run); - union_must_literal_sets(&mut must, extract_must_literal_set(expr)); + flush_literal_run(&mut must_literals, &mut needles, &mut literal_run); + union_literal_sets(&mut must_literals, child.must_literals); + union_literal_sets(&mut needles, child.needles); + } + + flush_literal_run(&mut must_literals, &mut needles, &mut literal_run); + AstAnalysisSet { + must_literals, + needles, + nullable, } +} + +fn analyze_alternate(left: &Ast, right: &Ast) -> AstAnalysisSet { + let left = analyze_ast_set(left); + let right = analyze_ast_set(right); - flush_literal_run(&mut must, &mut literal_run); - prune_must_literal_set(&mut must); - must + let must_literals = intersect_literal_sets(left.must_literals, right.must_literals); + let mut needles = left.needles; + union_literal_sets(&mut needles, right.needles); + + AstAnalysisSet { + must_literals, + needles, + nullable: left.nullable || right.nullable, + } } fn ast_single_literal(ast: &Ast) -> Option { @@ -197,35 +283,39 @@ fn class_single_literal(class: &CharClass) -> Option { } } -fn flush_literal_run(set: &mut BTreeSet, literal_run: &mut String) { +fn flush_literal_run( + must_literals: &mut BTreeSet, + needles: &mut BTreeSet, + literal_run: &mut String, +) { if literal_run.is_empty() { return; } - set.insert(std::mem::take(literal_run)); - prune_must_literal_set(set); + let literal = std::mem::take(literal_run); + must_literals.insert(literal.clone()); + prune_literal_set(must_literals); + needles.insert(literal); + prune_literal_set(needles); } -fn union_must_literal_sets(dst: &mut BTreeSet, src: BTreeSet) { +fn union_literal_sets(dst: &mut BTreeSet, src: BTreeSet) { dst.extend(src); - prune_must_literal_set(dst); + prune_literal_set(dst); } -fn intersect_must_literal_sets( - left: BTreeSet, - right: BTreeSet, -) -> BTreeSet { +fn intersect_literal_sets(left: BTreeSet, right: BTreeSet) -> BTreeSet { let mut intersection = BTreeSet::new(); for literal in left { if right.contains(&literal) { intersection.insert(literal); } } - prune_must_literal_set(&mut intersection); + prune_literal_set(&mut intersection); intersection } -fn prune_must_literal_set(set: &mut BTreeSet) { +fn prune_literal_set(set: &mut BTreeSet) { if set.len() <= MUST_LITERAL_LIMIT { return; } @@ -236,7 +326,7 @@ fn prune_must_literal_set(set: &mut BTreeSet) { *set = literals.into_iter().collect(); } -fn must_literal_set_to_vec(set: BTreeSet) -> Vec { +fn literal_set_to_vec(set: BTreeSet) -> Vec { let mut literals: Vec = set.into_iter().collect(); literals.sort_by(|a, b| compare_literals(a, b)); literals.truncate(MUST_LITERAL_LIMIT); @@ -251,7 +341,9 @@ fn compare_literals(a: &str, b: &str) -> Ordering { #[cfg(test)] mod tests { - use super::{MUST_LITERAL_LIMIT, extract_must_literals}; + use super::{ + MUST_LITERAL_LIMIT, analyze_ast, extract_must_literals, extract_needles, is_nullable, + }; use crate::engine::parser::parse; #[test] @@ -302,4 +394,61 @@ mod tests { .collect::>(); assert_eq!(actual, expected); } + + #[test] + fn test_analyze_ast_alternate_needles_and_nullable() { + let ast = parse("(abc|def)").unwrap(); + let actual = analyze_ast(&ast); + assert_eq!(actual.needles, vec!["abc".to_string(), "def".to_string()]); + assert!(!actual.nullable); + } + + #[test] + fn test_analyze_ast_zero_or_more_nullable_and_needles() { + let ast = parse("(abc)*").unwrap(); + let actual = analyze_ast(&ast); + assert!(actual.nullable); + assert_eq!(actual.needles, vec!["abc".to_string()]); + } + + #[test] + fn test_is_nullable_repeat_zero_to_n() { + let ast = parse("(abc|def){0,3}").unwrap(); + assert!(is_nullable(&ast)); + } + + #[test] + fn test_is_nullable_ab_plus_c() { + let ast = parse("ab+c").unwrap(); + assert!(!is_nullable(&ast)); + } + + #[test] + fn test_extract_needles_limit_prefers_longer_then_lexicographic() { + let pattern = [ + "ppp", "aaa", "qqq", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii", "jjj", + "kkk", "lll", "mmm", "nnn", "ooo", "zzzz", + ] + .join("|"); + let ast = parse(&pattern).unwrap(); + + let actual = extract_needles(&ast); + assert_eq!(actual.len(), MUST_LITERAL_LIMIT); + let expected = [ + "zzzz", "aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii", "jjj", "kkk", + "lll", "mmm", "nnn", "ooo", + ] + .into_iter() + .map(str::to_string) + .collect::>(); + assert_eq!(actual, expected); + } + + #[test] + fn test_backreference_analysis_is_conservative() { + let ast = parse("\\1").unwrap(); + let actual = analyze_ast(&ast); + assert!(!actual.nullable); + assert!(actual.needles.is_empty()); + } } From e305ab1ff545bfd09b6ea87c3646f1d15c19292c Mon Sep 17 00:00:00 2001 From: shu-kitamura Date: Sat, 14 Feb 2026 00:35:44 +0900 Subject: [PATCH 7/8] feat: use nullable and needles in match --- crates/regex-core/src/engine.rs | 52 ++++++++-- crates/regex-core/src/engine/evaluator.rs | 34 ++++++- crates/regex-core/src/lib.rs | 111 +++++++++++++++++++++- 3 files changed, 184 insertions(+), 13 deletions(-) diff --git a/crates/regex-core/src/engine.rs b/crates/regex-core/src/engine.rs index a973aed..225f7af 100644 --- a/crates/regex-core/src/engine.rs +++ b/crates/regex-core/src/engine.rs @@ -7,9 +7,13 @@ mod parser; use thiserror::Error; -use crate::engine::{compiler::compile, evaluator::eval, parser::parse}; +use crate::engine::{ + compiler::compile, + evaluator::{eval, eval_from_starts}, + parser::parse, +}; -pub(crate) use ast::{Ast, analyze_ast}; +pub(crate) use ast::{Ast, AstAnalysis, analyze_ast}; pub use compiler::CompileError; pub use evaluator::EvalError; pub use instruction::Instruction; @@ -57,13 +61,22 @@ where } } -/// Parse, extract must literals, and compile a pattern. -pub(crate) fn compile_pattern_with_must_literals( +/// Parse, analyze, and compile a pattern. +pub(crate) fn compile_pattern_with_analysis( pattern: &str, -) -> Result<(Vec, Vec), RegexError> { +) -> Result<(Vec, AstAnalysis), RegexError> { let ast: Ast = parse(pattern)?; let analysis = analyze_ast(&ast); let instructions = compile(&ast)?; + Ok((instructions, analysis)) +} + +/// Parse, extract must literals, and compile a pattern. +#[allow(dead_code)] +pub(crate) fn compile_pattern_with_must_literals( + pattern: &str, +) -> Result<(Vec, Vec), RegexError> { + let (instructions, analysis) = compile_pattern_with_analysis(pattern)?; Ok((instructions, analysis.must_literals)) } @@ -72,11 +85,21 @@ pub fn match_line(code: &[Instruction], line: &str) -> Result Ok(eval(code, line)?) } +/// Match an instruction sequence from provided starting character indices. +pub(crate) fn match_line_from_starts( + code: &[Instruction], + line: &str, + starts: &[usize], +) -> Result { + Ok(eval_from_starts(code, line, starts)?) +} + #[cfg(test)] mod tests { use crate::engine::{ - CompileError, RegexError, compile_pattern_with_must_literals, instruction::Instruction, - match_line, + CompileError, RegexError, compile_pattern_with_analysis, + compile_pattern_with_must_literals, instruction::Instruction, match_line, + match_line_from_starts, }; #[test] @@ -107,4 +130,19 @@ mod tests { let (_code, must_literals) = compile_pattern_with_must_literals(".*abc.*").unwrap(); assert_eq!(must_literals, vec!["abc".to_string()]); } + + #[test] + fn test_compile_pattern_with_analysis() { + let (_code, analysis) = compile_pattern_with_analysis("(abc|def)").unwrap(); + assert!(analysis.must_literals.is_empty()); + assert_eq!(analysis.needles, vec!["abc".to_string(), "def".to_string()]); + assert!(!analysis.nullable); + } + + #[test] + fn test_match_line_from_starts() { + let (code, _) = compile_pattern_with_must_literals("abc").unwrap(); + assert!(!match_line_from_starts(&code, "xabc", &[0]).unwrap()); + assert!(match_line_from_starts(&code, "xabc", &[1]).unwrap()); + } } diff --git a/crates/regex-core/src/engine/evaluator.rs b/crates/regex-core/src/engine/evaluator.rs index 8e8a213..b45818e 100644 --- a/crates/regex-core/src/engine/evaluator.rs +++ b/crates/regex-core/src/engine/evaluator.rs @@ -251,6 +251,29 @@ fn eval_from_start_inner( Ok(false) } +/// Evaluates whether `input` matches from any start index in `starts`. +pub(crate) fn eval_from_starts( + inst: &[Instruction], + input: &str, + starts: &[usize], +) -> Result { + let chars: Vec = input.chars().collect(); + let capture_slots = max_capture_index(inst) + .checked_add(1) + .ok_or(EvalError::PCOverFlow)?; + + for start in starts { + if *start > chars.len() { + continue; + } + if eval_from_start_inner(inst, &chars, *start, capture_slots)? { + return Ok(true); + } + } + + Ok(false) +} + /// Evaluates whether `input` matches at any starting position. pub fn eval(inst: &[Instruction], input: &str) -> Result { let chars: Vec = input.chars().collect(); @@ -272,7 +295,7 @@ mod tests { use crate::engine::{ ast::{CharClass, CharRange, Predicate}, compiler::compile, - evaluator::{EvalError, eval}, + evaluator::{EvalError, eval, eval_from_starts}, instruction::Instruction, parser::parse, }; @@ -341,4 +364,13 @@ mod tests { let actual = eval(&inst, "abc"); assert_eq!(actual, Err(EvalError::InvalidPC)); } + + #[test] + fn test_eval_from_starts_uses_only_provided_positions() { + let ast = parse("abc").unwrap(); + let inst = compile(&ast).unwrap(); + + assert!(!eval_from_starts(&inst, "xabc", &[0]).unwrap()); + assert!(eval_from_starts(&inst, "xabc", &[1]).unwrap()); + } } diff --git a/crates/regex-core/src/lib.rs b/crates/regex-core/src/lib.rs index fd24521..4e12779 100644 --- a/crates/regex-core/src/lib.rs +++ b/crates/regex-core/src/lib.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use engine::Instruction; mod engine; @@ -9,6 +11,12 @@ pub struct Regex { code: Vec, /// Must-have literal substrings used for a fast pre-filter. must_literals: Vec, + /// Candidate literal substrings used to find likely start positions. + needles: Vec, + /// Whether this pattern can match the empty string. + nullable: bool, + /// Whether the instruction stream contains zero-width assertions. + has_assertion: bool, /// Enables case-insensitive matching by lowercasing pattern/input. is_ignore_case: bool, /// Inverts the final match result. @@ -22,15 +30,21 @@ impl Regex { is_ignore_case: bool, is_invert_match: bool, ) -> Result { - let (code, must_literals) = if is_ignore_case { - engine::compile_pattern_with_must_literals(&pattern.to_lowercase())? + let (code, analysis) = if is_ignore_case { + engine::compile_pattern_with_analysis(&pattern.to_lowercase())? } else { - engine::compile_pattern_with_must_literals(pattern)? + engine::compile_pattern_with_analysis(pattern)? }; + let has_assertion = code + .iter() + .any(|instruction| matches!(instruction, Instruction::Assert(_))); Ok(Self { code, - must_literals, + must_literals: analysis.must_literals, + needles: analysis.needles, + nullable: analysis.nullable, + has_assertion, is_ignore_case, is_invert_match, }) @@ -47,8 +61,12 @@ impl Regex { Ok(is_match ^ self.is_invert_match) } - /// Matches a line, optionally using a must-literal pre-filter first. + /// Matches a line with nullable/must/needle prefilters and a full-eval fallback. fn is_match_line(&self, line: &str) -> Result { + if self.nullable && !self.has_assertion { + return Ok(true); + } + if !self .must_literals .iter() @@ -57,8 +75,51 @@ impl Regex { return Ok(false); } + if self.must_literals.is_empty() && !self.needles.is_empty() { + let starts = Self::collect_start_positions_from_needles(line, &self.needles); + if !starts.is_empty() && engine::match_line_from_starts(&self.code, line, &starts)? { + return Ok(true); + } + } + engine::match_line(&self.code, line) } + + fn collect_start_positions_from_needles(line: &str, needles: &[String]) -> Vec { + let mut byte_starts = BTreeSet::new(); + for needle in needles { + if needle.is_empty() { + continue; + } + for (byte_start, _) in line.match_indices(needle) { + byte_starts.insert(byte_start); + } + } + + if byte_starts.is_empty() { + return Vec::new(); + } + + let mut starts = Vec::with_capacity(byte_starts.len()); + let mut targets = byte_starts.into_iter().peekable(); + let mut char_index = 0; + for (byte_index, _) in line.char_indices() { + while let Some(target) = targets.peek() { + if *target == byte_index { + starts.push(char_index); + targets.next(); + } else { + break; + } + } + if targets.peek().is_none() { + break; + } + char_index += 1; + } + + starts + } } #[cfg(test)] @@ -124,9 +185,15 @@ mod tests { fn test_extracts_must_literals_for_filtering() { let regex = Regex::new(".*abc.*", false, false).unwrap(); assert_eq!(regex.must_literals, vec!["abc".to_string()]); + assert_eq!(regex.needles, vec!["abc".to_string()]); + assert!(!regex.nullable); let regex = Regex::new("ab*c", false, false).unwrap(); assert_eq!(regex.must_literals, vec!["a".to_string(), "c".to_string()]); + assert_eq!( + regex.needles, + vec!["a".to_string(), "b".to_string(), "c".to_string()] + ); } #[test] @@ -155,4 +222,38 @@ mod tests { assert!(regex.is_match("def").unwrap()); assert!(!regex.is_match("xyz").unwrap()); } + + #[test] + fn test_nullable_fast_path_without_assertion() { + let regex = Regex::new("a*", false, false).unwrap(); + assert!(regex.nullable); + assert!(!regex.has_assertion); + assert!(regex.is_match("zzz").unwrap()); + } + + #[test] + fn test_nullable_fast_path_is_guarded_by_assertion() { + let regex = Regex::new("^$", false, false).unwrap(); + assert!(regex.nullable); + assert!(regex.has_assertion); + assert!(regex.is_match("").unwrap()); + assert!(!regex.is_match("x").unwrap()); + } + + #[test] + fn test_needles_preferred_search_still_matches() { + let regex = Regex::new("(abc|def)", false, false).unwrap(); + assert!(regex.must_literals.is_empty()); + assert_eq!(regex.needles, vec!["abc".to_string(), "def".to_string()]); + assert!(regex.is_match("xyzdef").unwrap()); + assert!(!regex.is_match("xyz").unwrap()); + } + + #[test] + fn test_needles_fallback_to_full_scan_preserves_correctness() { + let regex = Regex::new("(a|[0-9])", false, false).unwrap(); + assert!(regex.must_literals.is_empty()); + assert_eq!(regex.needles, vec!["a".to_string()]); + assert!(regex.is_match("5").unwrap()); + } } From 00a842dc9e5515b63dd1235907108255ae780458 Mon Sep 17 00:00:00 2001 From: shu-kitamura Date: Sat, 14 Feb 2026 00:44:18 +0900 Subject: [PATCH 8/8] fix clippy --- crates/regex-core/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/regex-core/src/lib.rs b/crates/regex-core/src/lib.rs index 4e12779..932d030 100644 --- a/crates/regex-core/src/lib.rs +++ b/crates/regex-core/src/lib.rs @@ -102,8 +102,7 @@ impl Regex { let mut starts = Vec::with_capacity(byte_starts.len()); let mut targets = byte_starts.into_iter().peekable(); - let mut char_index = 0; - for (byte_index, _) in line.char_indices() { + for (char_index, (byte_index, _)) in line.char_indices().enumerate() { while let Some(target) = targets.peek() { if *target == byte_index { starts.push(char_index); @@ -115,7 +114,6 @@ impl Regex { if targets.peek().is_none() { break; } - char_index += 1; } starts