Integrate tree-sitter queries into the search

src/cli/repl/commands/repl-query.ts

@@ -76,7 +76,7 @@ async function processQueryArgs(line: string, parser: KnownParser, output: ReplO
 	const processed = await getDataflow(config, parser, args.join(' '));
 	return {
 		parsedQuery,
-		query: await Promise.resolve(executeQueries({ dataflow: processed.dataflow, ast: processed.normalize, config }, parsedQuery)),
+		query: await Promise.resolve(executeQueries({ parse: processed.parse, dataflow: processed.dataflow, ast: processed.normalize, config }, parsedQuery)),
 		processed
 	};
 }

src/cli/repl/server/connection.ts

@@ -375,10 +375,11 @@ export class FlowRServerConnection {
 			return;
 		}
 
-		const { dataflow: dfg, normalize: ast } = fileInformation.pipeline.getResults(true);
+		const { parse, dataflow: dfg, normalize: ast } = fileInformation.pipeline.getResults(true);
+		guard(parse !== undefined, `Parse results must be present (request: ${request.filetoken})`);
 		guard(dfg !== undefined, `Dataflow graph must be present (request: ${request.filetoken})`);
 		guard(ast !== undefined, `AST must be present (request: ${request.filetoken})`);
-		void Promise.resolve(executeQueries({ dataflow: dfg, ast, config: this.config }, request.query)).then(results => {
+		void Promise.resolve(executeQueries({ parse, dataflow: dfg, ast, config: this.config }, request.query)).then(results => {
 			sendMessage<QueryResponseMessage>(this.socket, {
 				type: 'response-query',
 				id:   request.id,

src/core/steps/all/core/01-parse-tree-sitter.ts

@@ -5,18 +5,6 @@ import type { Tree } from 'web-tree-sitter';
 import type { ParseRequiredInput } from '../../../../r-bridge/parser';
 import { parseRequests } from '../../../../r-bridge/parser';
 
-export interface ParseStepOutputTS {
-    readonly parsed: Tree
-}
-
-export interface TreeSitterParseJson {
-	readonly '.meta': {
-		readonly tokenCount:           number,
-		readonly tokenCountNoComments: number
-	},
-	readonly str: string
-}
-
 export const PARSE_WITH_TREE_SITTER_STEP = {
 	name:              'parse',
 	humanReadableName: 'parse with tree-sitter',

src/core/steps/all/core/11-normalize-tree-sitter.ts

@@ -11,11 +11,12 @@ import type { DeepReadonly } from 'ts-essentials';
 import { normalizeTreeSitter } from '../../../../r-bridge/lang-4.x/ast/parser/json/parser';
 import type { NormalizeRequiredInput } from './10-normalize';
 import { getCurrentRequestFile } from './10-normalize';
-import type { ParseStepOutputTS } from './01-parse-tree-sitter';
 import type { FlowrConfigOptions } from '../../../../config';
+import type { ParseStepOutput } from '../../../../r-bridge/parser';
+import type { Tree } from 'web-tree-sitter';
 
-function processor(results: { 'parse'?: ParseStepOutputTS }, input: Partial<NormalizeRequiredInput>, config: FlowrConfigOptions) {
-	return normalizeTreeSitter(results['parse'] as ParseStepOutputTS, input.getId, config, input.overwriteFilePath ?? getCurrentRequestFile(input.request));
+function processor(results: { 'parse'?: ParseStepOutput<Tree> }, input: Partial<NormalizeRequiredInput>, config: FlowrConfigOptions) {
+	return normalizeTreeSitter(results['parse'] as ParseStepOutput<Tree>, input.getId, config, input.overwriteFilePath ?? getCurrentRequestFile(input.request));
 }
 
 export const NORMALIZE_TREE_SITTER = {

src/documentation/doc-util/doc-query.ts

@@ -34,6 +34,7 @@ export async function showQuery<
 		request: requestFromInput(code)
 	}, defaultConfigOptions).allRemainingSteps();
 	const results = await Promise.resolve(executeQueries({
+		parse:    analysis.parse,
 		dataflow: analysis.dataflow,
 		ast:      analysis.normalize,
 		config:   cloneConfig(defaultConfigOptions)

src/linter/linter-executor.ts

@@ -7,8 +7,9 @@ import { runSearch } from '../search/flowr-search-executor';
 import type { DeepPartial } from 'ts-essentials';
 import { deepMergeObject } from '../util/objects';
 import type { FlowrConfigOptions } from '../config';
+import type { KnownParserType, ParseStepOutput } from '../r-bridge/parser';
 
-export function executeLintingRule<Name extends LintingRuleNames>(ruleName: Name, input: { normalize: NormalizedAst, dataflow: DataflowInformation, config: FlowrConfigOptions }, lintingRuleConfig?: DeepPartial<LintingRuleConfig<Name>>): LintingResults<Name> {
+export function executeLintingRule<Name extends LintingRuleNames>(ruleName: Name, input: { parse: ParseStepOutput<KnownParserType>, normalize: NormalizedAst, dataflow: DataflowInformation, config: FlowrConfigOptions }, lintingRuleConfig?: DeepPartial<LintingRuleConfig<Name>>): LintingResults<Name> {
 	try {
 		const rule = LintingRules[ruleName] as unknown as LintingRule<LintingRuleResult<Name>, LintingRuleMetadata<Name>, LintingRuleConfig<Name>>;
 		const fullConfig = deepMergeObject<LintingRuleConfig<Name>>(rule.info.defaultConfig, lintingRuleConfig);

src/queries/base-query-format.ts

@@ -2,6 +2,7 @@ import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/de
 import type { DataflowInformation } from '../dataflow/info';
 import type { FlowrConfigOptions } from '../config';
 import type { SemVer } from 'semver';
+import type { KnownParserType } from '../r-bridge/parser';
 
 export interface BaseQueryFormat {
 	/** used to select the query type :) */
@@ -17,8 +18,9 @@ export interface BaseQueryResult {
 }
 
 export interface BasicQueryData {
-    readonly lib?:  Record<string, SemVer>;
-	readonly ast:      NormalizedAst;
-	readonly dataflow: DataflowInformation;
-	readonly config:   FlowrConfigOptions;
+    readonly lib?:     Record<string, SemVer>;
+    readonly parse:    { parsed: KnownParserType };
+    readonly ast:      NormalizedAst;
+    readonly dataflow: DataflowInformation;
+    readonly config:   FlowrConfigOptions;
 }

src/queries/catalog/linter-query/linter-query-executor.ts

@@ -6,7 +6,7 @@ import { log } from '../../../util/log';
 import type { ConfiguredLintingRule } from '../../../linter/linter-format';
 import { executeLintingRule } from '../../../linter/linter-executor';
 
-export function executeLinterQuery({ ast, dataflow, config }: BasicQueryData, queries: readonly LinterQuery[]): LinterQueryResult {
+export function executeLinterQuery({ parse, ast, dataflow, config }: BasicQueryData, queries: readonly LinterQuery[]): LinterQueryResult {
 	const flattened = queries.flatMap(q => q.rules ?? (Object.keys(LintingRules) as LintingRuleNames[]));
 	const distinct = new Set(flattened);
 	if(distinct.size !== flattened.length) {
@@ -18,7 +18,7 @@ export function executeLinterQuery({ ast, dataflow, config }: BasicQueryData, qu
 
 	const start = Date.now();
 
-	const input = { normalize: ast, dataflow, config };
+	const input = { parse, normalize: ast, dataflow, config };
 	for(const entry of distinct) {
 		const ruleName = typeof entry === 'string' ? entry : entry.name;
 		results.results[ruleName] = executeLintingRule<typeof ruleName>(ruleName, input, (entry as ConfiguredLintingRule)?.config);	

src/queries/catalog/search-query/search-query-executor.ts

@@ -4,13 +4,13 @@ import { runSearch } from '../../../search/flowr-search-executor';
 import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id';
 import type { FlowrSearch } from '../../../search/flowr-search-builder';
 
-export function executeSearch({ ast, dataflow, config }: BasicQueryData, queries: readonly SearchQuery[]): SearchQueryResult {
+export function executeSearch({ parse, ast, dataflow, config }: BasicQueryData, queries: readonly SearchQuery[]): SearchQueryResult {
 	const start = Date.now();
 	const results: { ids: NodeId[], search: FlowrSearch }[] = [];
 	for(const query of queries) {
 		const { search } = query;
 		results.push({
-			ids: runSearch(search, { normalize: ast, dataflow, config } )
+			ids: runSearch(search, { parse, normalize: ast, dataflow, config } )
 				.getElements().map(({ node }) => node.info.id),
 			search
 		});

src/r-bridge/lang-4.x/ast/parser/json/parser.ts

@@ -5,11 +5,11 @@ import { decorateAst, deterministicCountingIdGenerator } from '../../model/proce
 import type { NoInfo, RNode } from '../../model/model';
 import { normalizeRootObjToAst } from '../main/internal/structure/normalize-root';
 import type { NormalizerData } from '../main/normalizer-data';
-import type { ParseStepOutputTS } from '../../../../../core/steps/all/core/01-parse-tree-sitter';
 import { normalizeTreeSitterTreeToAst } from '../../../tree-sitter/tree-sitter-normalize';
 import type { ParseStepOutput } from '../../../../parser';
 import type { FlowrConfigOptions } from '../../../../../config';
 import { getEngineConfig } from '../../../../../config';
+import type { Tree } from 'web-tree-sitter';
 
 export const parseLog = log.getSubLogger({ name: 'ast-parser' });
 
@@ -44,7 +44,7 @@ export function normalizeButNotDecorated(
  * Tree-Sitter pendant to {@link normalize}.
  */
 export function normalizeTreeSitter(
-	{ parsed }: ParseStepOutputTS,
+	{ parsed }: ParseStepOutput<Tree>,
 	getId: IdGenerator<NoInfo> = deterministicCountingIdGenerator(0),
 	config: FlowrConfigOptions,
 	file?: string

src/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor.ts

@@ -1,3 +1,4 @@
+import type { QueryCapture } from 'web-tree-sitter';
 import Parser from 'web-tree-sitter';
 
 import type { RParseRequest } from '../../retriever';
@@ -68,6 +69,12 @@ export class TreeSitterExecutor implements SyncParser<Parser.Tree> {
 		return this.parser.parse(sourceCode);
 	}
 
+	public query(source: string, tree: Parser.Tree): QueryCapture[] {
+		const query = this.parser.getLanguage().query(source);
+		const matches = query.matches(tree.rootNode);
+		return matches.flatMap(m => m.captures);
+	}
+
 	public close(): void {
 		this.parser.delete();
 	}

src/search/flowr-search.ts

@@ -6,6 +6,7 @@ import type { DataflowInformation } from '../dataflow/info';
 import type { FlowrConfigOptions } from '../config';
 import type { Enrichment, EnrichmentSearchArguments, EnrichmentData, EnrichmentElementContent, EnrichmentSearchContent, EnrichmentElementArguments } from './search-executor/search-enrichers';
 import { Enrichments } from './search-executor/search-enrichers';
+import type { KnownParserType, ParseStepOutput } from '../r-bridge/parser';
 
 /**
  * Yes, for now we do technically not need a wrapper around the RNode, but this allows us to attach caches etc.
@@ -52,9 +53,11 @@ export interface FlowrSearchGetFilter extends Record<string, unknown> {
 }
 
 type MinimumInputForFlowrSearch<P extends Pipeline> =
-    PipelineStepOutputWithName<P, 'normalize'> extends NormalizedAst ? (
-        PipelineStepOutputWithName<P, 'dataflow'> extends DataflowInformation ? PipelineOutput<P> & { normalize: NormalizedAst, dataflow: DataflowInformation, config: FlowrConfigOptions }
-            : never
+    PipelineStepOutputWithName<P, 'parse'> extends ParseStepOutput<KnownParserType> ? (
+        PipelineStepOutputWithName<P, 'normalize'> extends NormalizedAst ? (
+            PipelineStepOutputWithName<P, 'dataflow'> extends DataflowInformation ? PipelineOutput<P> & { parse: { parsed: KnownParserType }, normalize: NormalizedAst, dataflow: DataflowInformation, config: FlowrConfigOptions }
+                : never
+        ): never
     ): never
 
 /** we allow any pipeline, which provides us with a 'normalize' and 'dataflow' step */

src/search/search-executor/search-generators.ts

@@ -36,11 +36,12 @@
  * All supported generators!
  */
 export const generators = {
-	all:          generateAll,
-	get:          generateGet,
-	criterion:    generateCriterion,
-	from:         generateFrom,
-	'from-query': generateFromQuery
+	all:                      generateAll,
+	get:                      generateGet,
+	criterion:                generateCriterion,
+	from:                     generateFrom,
+	'from-query':             generateFromQuery,
+	'from-tree-sitter-query': generateFromTreeSitterQuery
 } as const;
 
 function generateAll(data: FlowrSearchInput<Pipeline>): FlowrSearchElements<ParentInformation> {
@@ -91,7 +92,7 @@
 }
 
 function generateFromQuery(data: FlowrSearchInput<Pipeline>, args: { from: readonly SynchronousQuery[] } ): FlowrSearchElements<ParentInformation, FlowrSearchElement<ParentInformation>[]> {
-	const result = executeQueries({ ast: data.normalize, dataflow: data.dataflow, config: data.config }, args.from);
+	const result = executeQueries({ parse: data.parse, ast: data.normalize, dataflow: data.dataflow, config: data.config }, args.from);
 
 	// collect involved nodes
 	const nodesByQuery = new Map<Query['type'], Set<FlowrSearchElement<ParentInformation>>>();
@@ -115,6 +116,15 @@
 	})) as unknown as FlowrSearchElements<ParentInformation, FlowrSearchElement<ParentInformation>[]>;
 }
 
+function generateFromTreeSitterQuery(data: FlowrSearchInput<Pipeline>, args: { source: string } ): FlowrSearchElements<ParentInformation, FlowrSearchElement<ParentInformation>[]> {
+	if(typeof data.parse.parsed === 'string') {
+		// TODO maybe a warning or something here?
+		return new FlowrSearchElements([]);
+	}
+	// TODO run query using TreeSitterExecutor and convert nodes to our ids using a map that we will generate in tree sitter normalization
+	return new FlowrSearchElements([]);
+}
+
 function generateCriterion(data: FlowrSearchInput<Pipeline>, args: { criterion: SlicingCriteria }): FlowrSearchElements<ParentInformation> {
 	return new FlowrSearchElements(
 		args.criterion.map(c => ({ node: data.normalize.idMap.get(slicingCriterionToId(c, data.normalize.idMap)) as RNodeWithParent }))

test/functionality/_helper/query.ts

@@ -72,7 +72,7 @@ export function assertQuery<
 			getId:   deterministicCountingIdGenerator(0)
 		}, defaultConfigOptions).allRemainingSteps();
 
-		const result = await Promise.resolve(executeQueries<Queries['type'], VirtualArguments>({ dataflow: info.dataflow, ast: info.normalize, config: defaultConfigOptions }, queries));
+		const result = await Promise.resolve(executeQueries<Queries['type'], VirtualArguments>({ parse: info.parse, dataflow: info.dataflow, ast: info.normalize, config: defaultConfigOptions }, queries));
 
 		log.info(`total query time: ${result['.meta'].timing.toFixed(0)}ms (~1ms accuracy)`);
 

test/functionality/_helper/shell.ts

@@ -58,7 +58,7 @@ import type { CfgProperty } from '../../../src/control-flow/cfg-properties';
 import { assertCfgSatisfiesProperties } from '../../../src/control-flow/cfg-properties';
 import type { FlowrConfigOptions } from '../../../src/config';
 import { cloneConfig, defaultConfigOptions } from '../../../src/config';
-import type { KnownParser } from '../../../src/r-bridge/parser';
+import type { KnownParser, KnownParserType, ParseStepOutput } from '../../../src/r-bridge/parser';
 import { SliceDirection } from '../../../src/core/steps/all/static-slicing/00-slice';
 
 export const testWithShell = (msg: string, fn: (shell: RShell, test: unknown) => void | Promise<void>) => {
@@ -364,7 +364,7 @@ export function assertDataflow<P extends Pipeline>(
 	name: string | TestLabel,
 	shell: RShell,
 	input: string | RParseRequests,
-	expected: DataflowGraph | ((data: PipelineOutput<P> & { normalize: NormalizedAst, dataflow: DataflowInformation }) => DataflowGraph),
+	expected: DataflowGraph | ((data: PipelineOutput<P> & { parse: ParseStepOutput<KnownParserType>, normalize: NormalizedAst, dataflow: DataflowInformation }) => DataflowGraph),
 	userConfig?: Partial<DataflowTestConfiguration>,
 	startIndexForDeterministicIds = 0,
 	config = cloneConfig(defaultConfigOptions)