fix(scanner): reject invalid regexes at compile time

AGENTS.md

@@ -13,3 +13,5 @@ When writing Go code, prefer early returns over nested conditionals. Keep the ha
 Never use `go build`. Use `go run .` instead.
 
 To swap the regex library, only change the import path and keep the `regexp` alias. For example, to switch from coregex to go-re2, change `regexp "github.com/coregx/coregex"` to `regexp "github.com/wasilibs/go-re2"`.
+
+Regexes are compiled lazily on first atom hit, and a compile error there counts as "no match". `checkRegex` therefore runs at compile time so invalid patterns (e.g. repetition above 1000) fail `Compile`: fast `regexp/syntax` parse, and only if that fails, the configured compiler decides (Go syntax rejects raw Latin-1 bytes and `\C`, which RE2 accepts). Check the final pattern (after `buildRE2Pattern`/`hexStringToRegex`), never the raw YARA one.

go.sum

@@ -10,22 +10,12 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA=
-github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU=
 github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU=
 github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0=
-github.com/wasilibs/go-re2 v1.10.0 h1:vQZEBYZOCA9jdBMmrO4+CvqyCj0x4OomXTJ4a5/urQ0=
-github.com/wasilibs/go-re2 v1.10.0/go.mod h1:k+5XqO2bCJS+QpGOnqugyfwC04nw0jaglmjrrkG8U6o=
-github.com/wasilibs/go-re2 v1.11.0 h1:9j4uCAJyO3IJHygeOdUJNzm6sX3P1+3xuKTbCITQrvc=
-github.com/wasilibs/go-re2 v1.11.0/go.mod h1:cDOlK/pNYGbGKRNa+NnXDJ1c8R63tWjnTjFKzoSbLv0=
 github.com/wasilibs/go-re2 v1.12.0 h1:sq3A6ZOqT90HYY25MD5/cG8Xv6uT2AhmPgBfkCfhp10=
 github.com/wasilibs/go-re2 v1.12.0/go.mod h1:2W+7GrrdO4NHv7ITHz8Yy3a1IxzjZCk8JR5zgTprxZs=
 github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb h1:gQ+ZV4wJke/EBKYciZ2MshEouEHFuinB85dY3f5s1q8=
 github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY=
-golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
-golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
-golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
 golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

scanner/compile.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"regexp/syntax"
 	"strings"
 
 	"github.com/sansecio/yargo/ahocorasick"
@@ -191,6 +192,14 @@ func compileRegex(rules *Rules, s *ast.StringDef, slot int32, ruleName string, a
 	default:
 		return allPatterns, nil
 	}
+	// The regex itself is compiled lazily on first use, which discards
+	// errors, so reject invalid patterns here.
+	if err := checkRegex(rePattern, opts.RegexCompiler); err != nil {
+		if opts.SkipInvalidRegex {
+			return allPatterns, nil
+		}
+		return nil, fmt.Errorf("rule %q string %s: invalid regex: %w", ruleName, s.Name, err)
+	}
 	atoms, hasAtoms := extractAtoms(rePattern, minAtomLength)
 	requiresFullScan := !hasAtoms || caseInsensitive
 	if requiresFullScan {
@@ -259,6 +268,19 @@ func hexStringToBytes(h ast.HexString) []byte {
 	return result
 }
 
+// checkRegex reports whether compile would accept pattern. regexp/syntax
+// follows RE2 syntax (including the repetition limit of 1000) and parses in
+// microseconds, where a go-re2 compile takes a WASM round trip. It is
+// stricter than RE2 in places (raw Latin-1 bytes, \C), and compile may be a
+// custom engine, so compile has the final say on anything it rejects.
+func checkRegex(pattern string, compile CompileFunc) error {
+	if _, err := syntax.Parse(pattern, syntax.Perl); err == nil {
+		return nil
+	}
+	_, err := compile(pattern)
+	return err
+}
+
 func hexStringToRegex(h ast.HexString) string {
 	var sb strings.Builder
 

scanner/compile_test.go

@@ -2,6 +2,7 @@ package scanner
 
 import (
 	"slices"
+	"strings"
 	"testing"
 	"time"
 
@@ -262,6 +263,120 @@ func TestRegexCompilerPanicRecoverySkipInvalid(t *testing.T) {
 	}
 }
 
+func TestInvalidRegex(t *testing.T) {
+	tests := []struct {
+		name  string
+		value ast.StringValue
+	}{
+		{"repetition above RE2 limit", ast.RegexString{Pattern: `lskdjf[a-z0-9]{16,4000}xyz`}},
+		{"unbalanced group", ast.RegexString{Pattern: `lskdjf(xyz`}},
+		{"comma quantifier above limit", ast.RegexString{Pattern: `lskdjf.{,4000}xyz`}},
+		{"hex jump above limit", ast.HexString{Tokens: []ast.HexToken{
+			ast.HexByte{Value: 0x61},
+			ast.HexByte{Value: 0x62},
+			ast.HexByte{Value: 0x63},
+			ast.HexJump{Min: new(0), Max: new(2000)},
+			ast.HexByte{Value: 0x64},
+		}}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := &ast.RuleSet{
+				Rules: []*ast.Rule{
+					{
+						Name:      "bad_regex",
+						Strings:   []*ast.StringDef{{Name: "$s", Value: tt.value}},
+						Condition: ast.AnyOf{Pattern: "them"},
+					},
+				},
+			}
+
+			_, err := Compile(rs)
+			if err == nil {
+				t.Fatal("Compile() error = nil, want invalid regex error")
+			}
+			if !strings.Contains(err.Error(), "bad_regex") || !strings.Contains(err.Error(), "invalid regex") {
+				t.Errorf("Compile() error = %v, want rule name and \"invalid regex\"", err)
+			}
+
+			rules, err := CompileWithOptions(rs, CompileOptions{SkipInvalidRegex: true})
+			if err != nil {
+				t.Fatalf("CompileWithOptions(SkipInvalidRegex) error = %v", err)
+			}
+			if _, n := rules.Stats(); n != 0 {
+				t.Errorf("regex patterns = %d, want 0 (skipped)", n)
+			}
+		})
+	}
+}
+
+// TestRegexRejectedByGoSyntax covers patterns that regexp/syntax rejects but
+// the configured compiler accepts. They must compile, and match input if set.
+func TestRegexRejectedByGoSyntax(t *testing.T) {
+	tests := []struct {
+		name    string
+		pattern string
+		input   string
+	}{
+		{"raw Latin-1 byte", "abcd\xe9fgh", "xx abcd\xe9fgh yy"},
+		// atom extraction reads \C as a literal C, so only compilation is checked
+		{"RE2 any-byte escape", `abcd\Cfgh`, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := &ast.RuleSet{
+				Rules: []*ast.Rule{
+					{
+						Name:      "latin1_regex",
+						Strings:   []*ast.StringDef{{Name: "$s", Value: ast.RegexString{Pattern: tt.pattern}}},
+						Condition: ast.AnyOf{Pattern: "them"},
+					},
+				},
+			}
+
+			rules, err := Compile(rs)
+			if err != nil {
+				t.Fatalf("Compile() error = %v", err)
+			}
+			if _, n := rules.Stats(); n != 1 {
+				t.Fatalf("regex patterns = %d, want 1", n)
+			}
+			if tt.input == "" {
+				return
+			}
+			var matches MatchRules
+			if err := rules.ScanMem([]byte(tt.input), 0, 10*time.Second, &matches); err != nil {
+				t.Fatalf("ScanMem() error = %v", err)
+			}
+			if len(matches) != 1 {
+				t.Errorf("expected 1 match, got %d", len(matches))
+			}
+		})
+	}
+}
+
+func TestInvalidRegexCustomCompiler(t *testing.T) {
+	rs := &ast.RuleSet{
+		Rules: []*ast.Rule{
+			{
+				Name:      "custom_syntax",
+				Strings:   []*ast.StringDef{{Name: "$s", Value: ast.RegexString{Pattern: `lskdjf[a-z]{1,4000}`}}},
+				Condition: ast.AnyOf{Pattern: "them"},
+			},
+		},
+	}
+
+	// a compiler without RE2's repetition limit decides what is valid
+	_, err := CompileWithOptions(rs, CompileOptions{
+		RegexCompiler: func(pattern string) (Regexp, error) {
+			return experimental.CompileLatin1(`lskdjf[a-z]{1,40}`)
+		},
+	})
+	if err != nil {
+		t.Fatalf("CompileWithOptions() error = %v", err)
+	}
+}
+
 func TestCustomRegexCompiler(t *testing.T) {
 	rs := &ast.RuleSet{
 		Rules: []*ast.Rule{