diff --git a/CHANGELOG.md b/CHANGELOG.md index 6707a6499..5a7a3bc27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Changed +- **zeph-common**: extended `treesitter::lang_for_ext` to cover the full extension set + (`bash`/`sh`/`zsh`, `toml`, `json`/`jsonc`, `md`/`markdown` in addition to the existing + `rs`, `py`/`pyi`, `js`/`jsx`/`mjs`/`cjs`, `ts`/`tsx`/`mts`/`cts`, `go`) and made it the single + source of truth for extension-to-grammar mapping; `zeph-index::languages::{Lang::grammar, + detect_language}` and `zeph-tools::search_code::lang_info_for_path` now delegate to it + instead of each hand-rolling its own copy of the mapping (closes #5971). - **BREAKING**: `CommandHandler::requires_auth()` now defaults to `true` (fail-closed), reversing the previous fail-open default. This is the 4th recurrence of a slash-command handler silently exposing privileged behavior to remote channels (Telegram/Discord/Slack) diff --git a/Cargo.lock b/Cargo.lock index 7ac586266..95bc1c5fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10800,10 +10800,14 @@ dependencies = [ "tokio-util", "tracing", "tree-sitter", + "tree-sitter-bash", "tree-sitter-go", "tree-sitter-javascript", + "tree-sitter-json", + "tree-sitter-md", "tree-sitter-python", "tree-sitter-rust", + "tree-sitter-toml-ng", "tree-sitter-typescript", "url", "uuid", @@ -11027,14 +11031,10 @@ dependencies = [ "tokio", "tracing", "tree-sitter", - "tree-sitter-bash", "tree-sitter-go", "tree-sitter-javascript", - "tree-sitter-json", - "tree-sitter-md", "tree-sitter-python", "tree-sitter-rust", - "tree-sitter-toml-ng", "tree-sitter-typescript", "uuid", "zeph-common", @@ -11391,14 +11391,10 @@ dependencies = [ "toml 1.1.2+spec-1.1.0", "tracing", "tree-sitter", - "tree-sitter-bash", "tree-sitter-go", "tree-sitter-javascript", - "tree-sitter-json", - "tree-sitter-md", "tree-sitter-python", "tree-sitter-rust", - "tree-sitter-toml-ng", "tree-sitter-typescript", "unicode-normalization", "url", diff --git a/crates/zeph-common/Cargo.toml b/crates/zeph-common/Cargo.toml index 6c6199fc3..cd21054fa 100644 --- a/crates/zeph-common/Cargo.toml +++ b/crates/zeph-common/Cargo.toml @@ -18,10 +18,14 @@ http-middleware = ["dep:axum", "dep:subtle"] jsonschema = ["dep:schemars"] treesitter = [ "dep:tree-sitter", + "dep:tree-sitter-bash", "dep:tree-sitter-go", "dep:tree-sitter-javascript", + "dep:tree-sitter-json", + "dep:tree-sitter-md", "dep:tree-sitter-python", "dep:tree-sitter-rust", + "dep:tree-sitter-toml-ng", "dep:tree-sitter-typescript", ] @@ -40,10 +44,14 @@ tokio = { workspace = true, features = ["macros", "net", "rt", "rt-multi-thread" tokio-util.workspace = true tracing.workspace = true tree-sitter = { workspace = true, optional = true } +tree-sitter-bash = { workspace = true, optional = true } tree-sitter-go = { workspace = true, optional = true } tree-sitter-javascript = { workspace = true, optional = true } +tree-sitter-json = { workspace = true, optional = true } +tree-sitter-md = { workspace = true, optional = true } tree-sitter-python = { workspace = true, optional = true } tree-sitter-rust = { workspace = true, optional = true } +tree-sitter-toml-ng = { workspace = true, optional = true } tree-sitter-typescript = { workspace = true, optional = true } url.workspace = true uuid = { workspace = true, features = ["v4"] } diff --git a/crates/zeph-common/src/treesitter.rs b/crates/zeph-common/src/treesitter.rs index 938dc9d44..dfafd0c96 100644 --- a/crates/zeph-common/src/treesitter.rs +++ b/crates/zeph-common/src/treesitter.rs @@ -1,7 +1,10 @@ // SPDX-FileCopyrightText: 2026 Andrei G // SPDX-License-Identifier: MIT OR Apache-2.0 -//! Shared tree-sitter query constants and helpers used by zeph-tools and zeph-index. +//! Shared tree-sitter query constants and helpers used by zeph-tools and zeph-index, +//! including [`lang_for_ext`] — the single source of truth both crates call to map a +//! file extension to a tree-sitter `Language`, instead of each maintaining its own +//! extension-to-language match arms. //! //! Only available with the `treesitter` feature. @@ -72,6 +75,10 @@ pub fn compile_query(lang: &Language, source: &str, label: &str) -> Option Option { @@ -81,6 +88,52 @@ pub fn lang_for_ext(ext: &str) -> Option { "js" | "jsx" | "mjs" | "cjs" => Some(tree_sitter_javascript::LANGUAGE.into()), "ts" | "tsx" | "mts" | "cts" => Some(tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()), "go" => Some(tree_sitter_go::LANGUAGE.into()), + "sh" | "bash" | "zsh" => Some(tree_sitter_bash::LANGUAGE.into()), + "toml" => Some(tree_sitter_toml_ng::LANGUAGE.into()), + "json" | "jsonc" => Some(tree_sitter_json::LANGUAGE.into()), + "md" | "markdown" => Some(tree_sitter_md::LANGUAGE.into()), _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Every extension group `zeph-index` and `zeph-tools` route through + /// `lang_for_ext` must resolve to a grammar (#5971): the five languages + /// that already had call sites plus the four added when the two crates' + /// hand-rolled mappings were consolidated into this function. + #[test] + fn lang_for_ext_covers_all_extension_groups() { + let supported = [ + "rs", "py", "pyi", "js", "jsx", "mjs", "cjs", "ts", "tsx", "mts", "cts", "go", "sh", + "bash", "zsh", "toml", "json", "jsonc", "md", "markdown", + ]; + for ext in supported { + assert!( + lang_for_ext(ext).is_some(), + "expected .{ext} to be supported" + ); + } + } + + #[test] + fn lang_for_ext_unsupported_returns_none() { + assert!(lang_for_ext("xyz").is_none()); + assert!(lang_for_ext("").is_none()); + } + + /// Aliases within one extension group must resolve to the identical + /// `Language`, and distinct groups must not collide. + #[test] + fn lang_for_ext_aliases_match_within_group_and_differ_across_groups() { + assert_eq!(lang_for_ext("sh"), lang_for_ext("bash")); + assert_eq!(lang_for_ext("sh"), lang_for_ext("zsh")); + assert_eq!(lang_for_ext("json"), lang_for_ext("jsonc")); + assert_eq!(lang_for_ext("md"), lang_for_ext("markdown")); + assert_ne!(lang_for_ext("sh"), lang_for_ext("toml")); + assert_ne!(lang_for_ext("toml"), lang_for_ext("json")); + assert_ne!(lang_for_ext("json"), lang_for_ext("md")); + } +} diff --git a/crates/zeph-index/Cargo.toml b/crates/zeph-index/Cargo.toml index 11d18086b..f2c1c1edf 100644 --- a/crates/zeph-index/Cargo.toml +++ b/crates/zeph-index/Cargo.toml @@ -28,14 +28,10 @@ tracing.workspace = true tree-sitter.workspace = true # See https://github.com/bug-ops/zeph (workspace dependencies only contain versions) -tree-sitter-bash.workspace = true tree-sitter-go.workspace = true tree-sitter-javascript.workspace = true -tree-sitter-json.workspace = true -tree-sitter-md.workspace = true tree-sitter-python.workspace = true tree-sitter-rust.workspace = true -tree-sitter-toml-ng.workspace = true tree-sitter-typescript.workspace = true uuid = { workspace = true, features = ["v4"] } zeph-common = { workspace = true, features = ["treesitter"] } diff --git a/crates/zeph-index/src/languages.rs b/crates/zeph-index/src/languages.rs index 6f10aae39..08eac3512 100644 --- a/crates/zeph-index/src/languages.rs +++ b/crates/zeph-index/src/languages.rs @@ -25,7 +25,7 @@ use serde::{Deserialize, Serialize}; // ts-query source strings for symbol and method extraction. // Shared symbol queries are sourced from zeph-common::treesitter. use zeph_common::treesitter::{ - GO_SYM_Q, JS_SYM_Q, PYTHON_SYM_Q, RUST_SYM_Q, TS_SYM_Q, compile_query, + GO_SYM_Q, JS_SYM_Q, PYTHON_SYM_Q, RUST_SYM_Q, TS_SYM_Q, compile_query, lang_for_ext, }; const RUST_METHOD_Q: &str = " @@ -153,17 +153,21 @@ impl Lang { /// ``` #[must_use] pub fn grammar(self) -> Option { - match self { - Self::Rust => Some(tree_sitter_rust::LANGUAGE.into()), - Self::Python => Some(tree_sitter_python::LANGUAGE.into()), - Self::JavaScript => Some(tree_sitter_javascript::LANGUAGE.into()), - Self::TypeScript => Some(tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()), - Self::Go => Some(tree_sitter_go::LANGUAGE.into()), - Self::Bash => Some(tree_sitter_bash::LANGUAGE.into()), - Self::Toml => Some(tree_sitter_toml_ng::LANGUAGE.into()), - Self::Json => Some(tree_sitter_json::LANGUAGE.into()), - Self::Markdown => Some(tree_sitter_md::LANGUAGE.into()), - } + // Delegate to the shared extension-to-grammar mapping via each variant's + // canonical extension, instead of re-listing every `tree_sitter_*::LANGUAGE` + // construction here (that list already lives in `lang_for_ext`). + let ext = match self { + Self::Rust => "rs", + Self::Python => "py", + Self::JavaScript => "js", + Self::TypeScript => "ts", + Self::Go => "go", + Self::Bash => "sh", + Self::Toml => "toml", + Self::Json => "json", + Self::Markdown => "md", + }; + lang_for_ext(ext) } /// Compiled ts-query for extracting top-level symbols (name + visibility capture). @@ -319,6 +323,10 @@ impl std::fmt::Display for Lang { #[must_use] pub fn detect_language(path: &Path) -> Option { let ext = path.extension()?.to_str()?; + // `lang_for_ext` is the single source of truth for which extensions have + // tree-sitter support; gating on it here means this match can never accept + // an extension the shared grammar lookup would reject. + lang_for_ext(ext)?; match ext { "rs" => Some(Lang::Rust), "py" | "pyi" => Some(Lang::Python), @@ -399,6 +407,56 @@ mod tests { assert_eq!(detect_language(Path::new("file")), None); } + #[test] + fn detect_language_go() { + assert_eq!(detect_language(Path::new("main.go")), Some(Lang::Go)); + } + + /// Covers the four extension groups added to `lang_for_ext` by #5971 + /// (bash, toml, json, markdown) — previously only exercised indirectly + /// via `grammar_returns_some_for_all_langs`, never through `detect_language` + /// itself, leaving the extension-string match arms without direct coverage. + #[test] + fn detect_language_bash_variants() { + for ext in &["sh", "bash", "zsh"] { + let path = format!("file.{ext}"); + assert_eq!( + detect_language(Path::new(&path)), + Some(Lang::Bash), + "failed for .{ext}" + ); + } + } + + #[test] + fn detect_language_toml() { + assert_eq!(detect_language(Path::new("Cargo.toml")), Some(Lang::Toml)); + } + + #[test] + fn detect_language_json_variants() { + for ext in &["json", "jsonc"] { + let path = format!("file.{ext}"); + assert_eq!( + detect_language(Path::new(&path)), + Some(Lang::Json), + "failed for .{ext}" + ); + } + } + + #[test] + fn detect_language_markdown_variants() { + for ext in &["md", "markdown"] { + let path = format!("file.{ext}"); + assert_eq!( + detect_language(Path::new(&path)), + Some(Lang::Markdown), + "failed for .{ext}" + ); + } + } + #[test] fn entity_node_kinds_rust_includes_function_item() { let kinds = Lang::Rust.entity_node_kinds(); @@ -427,6 +485,37 @@ mod tests { assert!(Lang::Markdown.grammar().is_some()); } + /// `Lang::grammar()` was rewritten (#5971) to delegate to `lang_for_ext` via + /// each variant's canonical extension instead of constructing the + /// `tree_sitter_*::LANGUAGE` directly. `is_some()` alone cannot catch a + /// wrong-but-non-empty mapping (e.g. a variant accidentally wired to a + /// sibling extension's grammar); this asserts each variant's grammar is + /// identical to what `detect_language` + `lang_for_ext` resolve for that + /// variant's own extension, i.e. the delegation is wired correctly, not + /// just non-empty. + #[test] + fn grammar_matches_detect_language_for_canonical_extension() { + let cases = [ + (Lang::Rust, "main.rs"), + (Lang::Python, "script.py"), + (Lang::JavaScript, "app.js"), + (Lang::TypeScript, "app.ts"), + (Lang::Go, "main.go"), + (Lang::Bash, "script.sh"), + (Lang::Toml, "Cargo.toml"), + (Lang::Json, "data.json"), + (Lang::Markdown, "README.md"), + ]; + for (lang, path) in cases { + assert_eq!(detect_language(Path::new(path)), Some(lang)); + assert_eq!( + lang.grammar(), + lang_for_ext(Path::new(path).extension().unwrap().to_str().unwrap()), + "grammar() mismatch for {lang:?}" + ); + } + } + #[test] fn is_indexable_known_extension() { assert!(is_indexable(Path::new("src/main.rs"))); diff --git a/crates/zeph-tools/Cargo.toml b/crates/zeph-tools/Cargo.toml index 86288a21e..9d5776ada 100644 --- a/crates/zeph-tools/Cargo.toml +++ b/crates/zeph-tools/Cargo.toml @@ -33,14 +33,10 @@ tokio-util.workspace = true toml.workspace = true tracing.workspace = true tree-sitter.workspace = true -tree-sitter-bash.workspace = true tree-sitter-go.workspace = true tree-sitter-javascript.workspace = true -tree-sitter-json.workspace = true -tree-sitter-md.workspace = true tree-sitter-python.workspace = true tree-sitter-rust.workspace = true -tree-sitter-toml-ng.workspace = true tree-sitter-typescript.workspace = true unicode-normalization.workspace = true url.workspace = true diff --git a/crates/zeph-tools/src/search_code.rs b/crates/zeph-tools/src/search_code.rs index 1804fa4df..13fa63ea6 100644 --- a/crates/zeph-tools/src/search_code.rs +++ b/crates/zeph-tools/src/search_code.rs @@ -24,7 +24,7 @@ use crate::registry::{InvocationHint, ToolDef}; // --------------------------------------------------------------------------- use zeph_common::treesitter::{ - GO_SYM_Q, JS_SYM_Q, PYTHON_SYM_Q, RUST_SYM_Q, TS_SYM_Q, compile_query, + GO_SYM_Q, JS_SYM_Q, PYTHON_SYM_Q, RUST_SYM_Q, TS_SYM_Q, compile_query, lang_for_ext, }; struct LangInfo { @@ -34,36 +34,28 @@ struct LangInfo { fn lang_info_for_path(path: &Path) -> Option { let ext = path.extension()?.to_str()?; - match ext { + let grammar = lang_for_ext(ext)?; + let symbol_query = match ext { "rs" => { static Q: LazyLock> = LazyLock::new(|| { let lang: tree_sitter::Language = tree_sitter_rust::LANGUAGE.into(); compile_query(&lang, RUST_SYM_Q, "rust") }); - Some(LangInfo { - grammar: tree_sitter_rust::LANGUAGE.into(), - symbol_query: Q.as_ref(), - }) + Q.as_ref() } "py" | "pyi" => { static Q: LazyLock> = LazyLock::new(|| { let lang: tree_sitter::Language = tree_sitter_python::LANGUAGE.into(); compile_query(&lang, PYTHON_SYM_Q, "python") }); - Some(LangInfo { - grammar: tree_sitter_python::LANGUAGE.into(), - symbol_query: Q.as_ref(), - }) + Q.as_ref() } "js" | "jsx" | "mjs" | "cjs" => { static Q: LazyLock> = LazyLock::new(|| { let lang: tree_sitter::Language = tree_sitter_javascript::LANGUAGE.into(); compile_query(&lang, JS_SYM_Q, "javascript") }); - Some(LangInfo { - grammar: tree_sitter_javascript::LANGUAGE.into(), - symbol_query: Q.as_ref(), - }) + Q.as_ref() } "ts" | "tsx" | "mts" | "cts" => { static Q: LazyLock> = LazyLock::new(|| { @@ -71,39 +63,21 @@ fn lang_info_for_path(path: &Path) -> Option { tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(); compile_query(&lang, TS_SYM_Q, "typescript") }); - Some(LangInfo { - grammar: tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), - symbol_query: Q.as_ref(), - }) + Q.as_ref() } "go" => { static Q: LazyLock> = LazyLock::new(|| { let lang: tree_sitter::Language = tree_sitter_go::LANGUAGE.into(); compile_query(&lang, GO_SYM_Q, "go") }); - Some(LangInfo { - grammar: tree_sitter_go::LANGUAGE.into(), - symbol_query: Q.as_ref(), - }) + Q.as_ref() } - "sh" | "bash" | "zsh" => Some(LangInfo { - grammar: tree_sitter_bash::LANGUAGE.into(), - symbol_query: None, - }), - "toml" => Some(LangInfo { - grammar: tree_sitter_toml_ng::LANGUAGE.into(), - symbol_query: None, - }), - "json" | "jsonc" => Some(LangInfo { - grammar: tree_sitter_json::LANGUAGE.into(), - symbol_query: None, - }), - "md" | "markdown" => Some(LangInfo { - grammar: tree_sitter_md::LANGUAGE.into(), - symbol_query: None, - }), _ => None, - } + }; + Some(LangInfo { + grammar, + symbol_query, + }) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -892,6 +866,55 @@ mod tests { } } + /// `lang_info_for_path` (#5971) now resolves its `grammar` field via the + /// shared `lang_for_ext`, and only the symbol-query match is left local to + /// this file. Before this test the only extension exercised through + /// `lang_info_for_path` was `.rs` (via `search_code_finds_structural_symbol`); + /// the other eight groups compiled but had no direct coverage at all in + /// this crate. Also checks that formats with no symbol query (bash, toml, + /// json, markdown) still resolve a grammar but leave `symbol_query` unset. + #[test] + fn lang_info_for_path_covers_all_extension_groups() { + let with_symbol_query = [ + "main.rs", + "script.py", + "app.js", + "app.jsx", + "app.ts", + "app.tsx", + "main.go", + ]; + for path in with_symbol_query { + let info = lang_info_for_path(Path::new(path)) + .unwrap_or_else(|| panic!("expected grammar for {path}")); + assert!( + info.symbol_query.is_some(), + "expected symbol_query for {path}" + ); + } + + let without_symbol_query = [ + "script.sh", + "script.bash", + "script.zsh", + "Cargo.toml", + "data.json", + "data.jsonc", + "README.md", + "README.markdown", + ]; + for path in without_symbol_query { + let info = lang_info_for_path(Path::new(path)) + .unwrap_or_else(|| panic!("expected grammar for {path}")); + assert!( + info.symbol_query.is_none(), + "expected no symbol_query for {path}" + ); + } + + assert!(lang_info_for_path(Path::new("file.xyz")).is_none()); + } + #[tokio::test] async fn search_code_requires_query_or_symbol() { let dir = tempfile::tempdir().unwrap();