[Query API] Clean-Up Query Definitions (#1065)

* refactor: basic SupportedQueries registry structure

* feat: allow registering an ascii summarizer for a query directly

* refactor: include query schemas in query definitions

* feat-fix: fixed meta queries being included in print

* refactor: move query defs into their own files

src/cli/repl/commands/repl-query.ts

@@ -7,7 +7,8 @@ import { splitAtEscapeSensitive } from '../../../util/args';
 import { italic } from '../../../util/ansi';
 import { describeSchema } from '../../../util/schema';
 import type { Query, QueryResults, SupportedQueryTypes } from '../../../queries/query';
-import { executeQueries } from '../../../queries/query';
+import {  executeQueries } from '../../../queries/query';
+
 import type { PipelineOutput } from '../../../core/steps/pipeline/pipeline';
 import { jsonReplacer } from '../../../util/json';
 import { AnyQuerySchema, QueriesSchema } from '../../../queries/query-schema';

src/documentation/doc-util/doc-query.ts

@@ -1,6 +1,7 @@
 import type { RShell } from '../../r-bridge/shell';
 import type { Queries, QueryResults, SupportedQueryTypes } from '../../queries/query';
-import { executeQueries } from '../../queries/query';
+import { SupportedQueries , executeQueries } from '../../queries/query';
+
 import { PipelineExecutor } from '../../core/pipeline-executor';
 import { DEFAULT_DATAFLOW_PIPELINE } from '../../core/steps/pipeline/default-pipelines';
 import { requestFromInput } from '../../r-bridge/retriever';
@@ -15,10 +16,7 @@ import { printAsMs } from '../../util/time';
 import type { NodeId } from '../../r-bridge/lang-4.x/ast/model/processing/node-id';
 import type { PipelineOutput } from '../../core/steps/pipeline/pipeline';
 import { BuiltIn } from '../../dataflow/environments/built-in';
-import { graphToMermaidUrl } from '../../util/mermaid/dfg';
-import { normalizedAstToMermaidUrl } from '../../util/mermaid/ast';
-import type { StaticSliceQuery } from '../../queries/catalog/static-slice-query/static-slice-query-format';
-import type { BaseQueryMeta } from '../../queries/base-query-format';
+import type { BaseQueryMeta, BaseQueryResult } from '../../queries/base-query-format';
 import { textWithTooltip } from './doc-hover-over';
 import type { CallContextQuerySubKindResult } from '../../queries/catalog/call-context-query/call-context-query-format';
 
@@ -61,7 +59,7 @@ function asciiCallContextSubHit(formatter: OutputFormatter, results: readonly Ca
 	return result.join(', ');
 }
 
-function asciiCallContext(formatter: OutputFormatter, results: QueryResults<'call-context'>['call-context'], processed: PipelineOutput<typeof DEFAULT_DATAFLOW_PIPELINE>): string {
+export function asciiCallContext(formatter: OutputFormatter, results: QueryResults<'call-context'>['call-context'], processed: PipelineOutput<typeof DEFAULT_DATAFLOW_PIPELINE>): string {
 	/* traverse over 'kinds' and within them 'subkinds' */
 	const result: string[] = [];
 	for(const [kind, { subkinds }] of Object.entries(results['kinds'])) {
@@ -73,7 +71,7 @@ function asciiCallContext(formatter: OutputFormatter, results: QueryResults<'cal
 	return result.join('\n');
 }
 
-function summarizeIdsIfTooLong(ids: readonly NodeId[]) {
+export function summarizeIdsIfTooLong(ids: readonly NodeId[]) {
 	const naive = ids.join(', ');
 	if(naive.length <= 20) {
 		return naive;
@@ -96,63 +94,9 @@ export function asciiSummaryOfQueryResult(formatter: OutputFormatter, totalInMs:
 		if(query === '.meta') {
 			continue;
 		}
-		if(query === 'call-context') {
-			const out = queryResults as QueryResults<'call-context'>['call-context'];
-			result.push(`Query: ${bold(query, formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
-			result.push(asciiCallContext(formatter, out, processed));
-			continue;
-		} else if(query === 'dataflow') {
-			const out = queryResults as QueryResults<'dataflow'>['dataflow'];
-			result.push(`Query: ${bold(query, formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
-			result.push(`   ╰ [Dataflow Graph](${graphToMermaidUrl(out.graph)})`);
-			continue;
-		} else if(query === 'id-map') {
-			const out = queryResults as QueryResults<'id-map'>['id-map'];
-			result.push(`Query: ${bold(query, formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
-			result.push(`   ╰ Id List: {${summarizeIdsIfTooLong([...out.idMap.keys()])}}`);
-			continue;
-		} else if(query === 'normalized-ast') {
-			const out = queryResults as QueryResults<'normalized-ast'>['normalized-ast'];
-			result.push(`Query: ${bold(query, formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
-			result.push(`   ╰ [Normalized AST](${normalizedAstToMermaidUrl(out.normalized.ast)})`);
-			continue;
-		} else if(query === 'static-slice') {
-			const out = queryResults as QueryResults<'static-slice'>['static-slice'];
-			result.push(`Query: ${bold(query, formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
-			for(const [fingerprint, obj] of Object.entries(out.results)) {
-				const { criteria, noMagicComments, noReconstruction } = JSON.parse(fingerprint) as StaticSliceQuery;
-				const addons = [];
-				if(noReconstruction) {
-					addons.push('no reconstruction');
-				}
-				if(noMagicComments) {
-					addons.push('no magic comments');
-				}
-				result.push(`   ╰ Slice for {${criteria.join(', ')}} ${addons.join(', ')}`);
-				if('reconstruct' in obj) {
-					result.push('     ╰ Code (newline as <code>&#92;n</code>): <code>' + obj.reconstruct.code.split('\n').join('\\n') + '</code>');
-				} else {
-					result.push(`     ╰ Id List: {${summarizeIdsIfTooLong([...obj.slice.result])}}`);
-				}
-			}
-			continue;
-		} else if(query === 'dataflow-cluster') {
-			const out = queryResults as QueryResults<'dataflow-cluster'>['dataflow-cluster'];
-			result.push(`Query: ${bold(query, formatter)} (${out['.meta'].timing.toFixed(0)}ms)`);
-			result.push(`   ╰ Found ${out.clusters.length} cluster${out.clusters.length === 1 ? '' : 's'}`);
-			for(const cluster of out.clusters) {
-				const unknownSideEffects = cluster.hasUnknownSideEffects ? '(has unknown side effect)' : '';
-				result.push(`      ╰ ${unknownSideEffects} {${summarizeIdsIfTooLong(cluster.members)}} ([marked](${
-					graphToMermaidUrl(processed.dataflow.graph, false, new Set(cluster.members))
-				}))`);
-			}
-			continue;
-		} else if(query === 'lineage') {
-			const out = queryResults as QueryResults<'lineage'>['lineage'];
-			result.push(`Query: ${bold(query, formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
-			for(const [criteria, lineage] of Object.entries(out.lineages)) {
-				result.push(`   ╰ ${criteria}: {${summarizeIdsIfTooLong([...lineage])}}`);
-			}
+
+		const queryType = SupportedQueries[query as SupportedQueryTypes];
+		if(queryType.asciiSummarizer(formatter, processed, queryResults as BaseQueryResult, result)) {
 			continue;
 		}
 

src/queries/catalog/call-context-query/call-context-query-format.ts

@@ -1,5 +1,11 @@
 import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
 import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id';
+import { executeCallContextQueries } from './call-context-query-executor';
+import { bold } from '../../../util/ansi';
+import { printAsMs } from '../../../util/time';
+import Joi from 'joi';
+import type { QueryResults, SupportedQuery } from '../../query';
+import { asciiCallContext } from '../../../documentation/doc-util/doc-query';
 
 export enum CallTargets {
 	/** call targets a function that is not defined locally (e.g., the call targets a library function) */
@@ -83,3 +89,25 @@ export interface CallContextQueryResult extends BaseQueryResult {
 
 export type CallContextQuery<CallName extends RegExp | string = RegExp | string> = DefaultCallContextQueryFormat<CallName> | SubCallContextQueryFormat<CallName>;
 
+export const CallContextQueryDefinition = {
+	executor:        executeCallContextQueries,
+	asciiSummarizer: (formatter, processed, queryResults, result) => {
+		const out = queryResults as QueryResults<'call-context'>['call-context'];
+		result.push(`Query: ${bold('call-context', formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
+		result.push(asciiCallContext(formatter, out, processed));
+		return true;
+	},
+	schema: Joi.object({
+		type:           Joi.string().valid('call-context').required().description('The type of the query.'),
+		callName:       Joi.string().required().description('Regex regarding the function name!'),
+		callNameExact:  Joi.boolean().optional().description('Should we automatically add the `^` and `$` anchors to the regex to make it an exact match?'),
+		kind:           Joi.string().optional().description('The kind of the call, this can be used to group calls together (e.g., linking `plot` to `visualize`). Defaults to `.`'),
+		subkind:        Joi.string().optional().description('The subkind of the call, this can be used to uniquely identify the respective call type when grouping the output (e.g., the normalized name, linking `ggplot` to `plot`). Defaults to `.`'),
+		callTargets:    Joi.string().valid(...Object.values(CallTargets)).optional().description('Call targets the function may have. This defaults to `any`. Request this specifically to gain all call targets we can resolve.'),
+		includeAliases: Joi.boolean().optional().description('Consider a case like `f <- function_of_interest`, do you want uses of `f` to be included in the results?'),
+		linkTo:         Joi.object({
+			type:     Joi.string().valid('link-to-last-call').required().description('The type of the linkTo sub-query.'),
+			callName: Joi.string().required().description('Regex regarding the function name of the last call. Similar to `callName`, strings are interpreted as a regular expression.')
+		}).optional().description('Links the current call to the last call of the given kind. This way, you can link a call like `points` to the latest graphics plot etc.')
+	}).description('Call context query used to find calls in the dataflow graph')
+} as const satisfies SupportedQuery<'call-context'>;

src/queries/catalog/cluster-query/cluster-query-format.ts

@@ -1,5 +1,11 @@
 import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
+import { bold } from '../../../util/ansi';
+import Joi from 'joi';
+import type { QueryResults, SupportedQuery } from '../../query';
 import type { DataflowGraphClusters } from '../../../dataflow/cluster';
+import { executeDataflowClusterQuery } from './cluster-query-executor';
+import { summarizeIdsIfTooLong } from '../../../documentation/doc-util/doc-query';
+import { graphToMermaidUrl } from '../../../util/mermaid/dfg';
 
 /**
  * Calculates and returns all clusters encountered in the dataflow graph.
@@ -12,3 +18,22 @@ export interface DataflowClusterQueryResult extends BaseQueryResult {
 	/** All clusters found in the respective dataflow */
 	readonly clusters: DataflowGraphClusters;
 }
+
+export const ClusterQueryDefinition = {
+	executor:        executeDataflowClusterQuery,
+	asciiSummarizer: (formatter, processed, queryResults, result) => {
+		const out = queryResults as QueryResults<'dataflow-cluster'>['dataflow-cluster'];
+		result.push(`Query: ${bold('dataflow-cluster', formatter)} (${out['.meta'].timing.toFixed(0)}ms)`);
+		result.push(`   ╰ Found ${out.clusters.length} cluster${out.clusters.length === 1 ? '' : 's'}`);
+		for(const cluster of out.clusters) {
+			const unknownSideEffects = cluster.hasUnknownSideEffects ? '(has unknown side effect)' : '';
+			result.push(`      ╰ ${unknownSideEffects} {${summarizeIdsIfTooLong(cluster.members)}} ([marked](${
+				graphToMermaidUrl(processed.dataflow.graph, false, new Set(cluster.members))
+			}))`);
+		}
+		return true;
+	},
+	schema: Joi.object({
+		type: Joi.string().valid('dataflow-cluster').required().description('The type of the query.'),
+	}).description('The cluster query calculates and returns all clusters in the dataflow graph.')
+} as const satisfies SupportedQuery<'dataflow-cluster'>;

src/queries/catalog/dataflow-query/dataflow-query-format.ts

@@ -1,5 +1,11 @@
 import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
 import type { DataflowGraph } from '../../../dataflow/graph/graph';
+import { executeDataflowQuery } from './dataflow-query-executor';
+import { bold } from '../../../util/ansi';
+import { printAsMs } from '../../../util/time';
+import { graphToMermaidUrl } from '../../../util/mermaid/dfg';
+import Joi from 'joi';
+import type { QueryResults, SupportedQuery } from '../../query';
 
 /**
  * Simple re-returns the dataflow graph of the analysis.
@@ -12,3 +18,16 @@ export interface DataflowQueryResult extends BaseQueryResult {
 	/** Please be aware that this is the graph in its JSON representation, use {@link DataflowGraph#fromJson} if the result is serialized */
 	readonly graph: DataflowGraph;
 }
+
+export const DataflowQueryDefinition = {
+	executor:        executeDataflowQuery,
+	asciiSummarizer: (formatter, _processed, queryResults, result) => {
+		const out = queryResults as QueryResults<'dataflow'>['dataflow'];
+		result.push(`Query: ${bold('dataflow', formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
+		result.push(`   ╰ [Dataflow Graph](${graphToMermaidUrl(out.graph)})`);
+		return true;
+	},
+	schema: Joi.object({
+		type: Joi.string().valid('dataflow').required().description('The type of the query.'),
+	}).description('The dataflow query simply returns the dataflow graph, there is no need to pass it multiple times!')
+} as const satisfies SupportedQuery<'dataflow'>;

src/queries/catalog/id-map-query/id-map-query-format.ts

@@ -1,5 +1,11 @@
 import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
 import type { AstIdMap } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate';
+import { executeIdMapQuery } from './id-map-query-executor';
+import { bold } from '../../../util/ansi';
+import { printAsMs } from '../../../util/time';
+import Joi from 'joi';
+import type { QueryResults, SupportedQuery } from '../../query';
+import { summarizeIdsIfTooLong } from '../../../documentation/doc-util/doc-query';
 
 export interface IdMapQuery extends BaseQueryFormat {
 	readonly type: 'id-map';
@@ -8,3 +14,16 @@ export interface IdMapQuery extends BaseQueryFormat {
 export interface IdMapQueryResult extends BaseQueryResult {
 	readonly idMap: AstIdMap;
 }
+
+export const IdMapQueryDefinition = {
+	executor:        executeIdMapQuery,
+	asciiSummarizer: (formatter, _processed, queryResults, result) => {
+		const out = queryResults as QueryResults<'id-map'>['id-map'];
+		result.push(`Query: ${bold('id-map', formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
+		result.push(`   ╰ Id List: {${summarizeIdsIfTooLong([...out.idMap.keys()])}}`);
+		return true;
+	},
+	schema: Joi.object({
+		type: Joi.string().valid('id-map').required().description('The type of the query.'),
+	}).description('The id map query retrieves the id map from the normalized AST.')
+} as const satisfies SupportedQuery<'id-map'>;

src/queries/catalog/lineage-query/lineage-query-format.ts

@@ -1,6 +1,12 @@
 import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
 import type { SingleSlicingCriterion } from '../../../slicing/criterion/parse';
 import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id';
+import type { QueryResults, SupportedQuery } from '../../query';
+import { bold } from '../../../util/ansi';
+import { printAsMs } from '../../../util/time';
+import Joi from 'joi';
+import { executeLineageQuery } from './lineage-query-executor';
+import { summarizeIdsIfTooLong } from '../../../documentation/doc-util/doc-query';
 
 /**
  * Calculates the lineage of the given criterion.
@@ -14,3 +20,19 @@ export interface LineageQueryResult extends BaseQueryResult {
 	/** Maps each criterion to the found lineage, duplicates are ignored. */
 	readonly lineages: Record<SingleSlicingCriterion, Set<NodeId>>;
 }
+
+export const LineageQueryDefinition = {
+	executor:        executeLineageQuery,
+	asciiSummarizer: (formatter, _processed, queryResults, result) => {
+		const out = queryResults as QueryResults<'lineage'>['lineage'];
+		result.push(`Query: ${bold('lineage', formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
+		for(const [criteria, lineage] of Object.entries(out.lineages)) {
+			result.push(`   ╰ ${criteria}: {${summarizeIdsIfTooLong([...lineage])}}`);
+		}
+		return true;
+	},
+	schema: Joi.object({
+		type: Joi.string().valid('lineage').required().description('The type of the query.'),
+		id:   Joi.string().required().description('The ID of the node to get the lineage of.')
+	}).description('Lineage query used to find the lineage of a node in the dataflow graph')
+} as const satisfies SupportedQuery<'lineage'>;

src/queries/catalog/normalized-ast-query/normalized-ast-query-format.ts

@@ -1,5 +1,11 @@
 import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
 import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate';
+import { executeNormalizedAstQuery } from './normalized-ast-query-executor';
+import { bold } from '../../../util/ansi';
+import { printAsMs } from '../../../util/time';
+import { normalizedAstToMermaidUrl } from '../../../util/mermaid/ast';
+import Joi from 'joi';
+import type { QueryResults, SupportedQuery } from '../../query';
 
 /**
  * Simple re-returns the normalized AST of the analysis.
@@ -11,3 +17,16 @@ export interface NormalizedAstQuery extends BaseQueryFormat {
 export interface NormalizedAstQueryResult extends BaseQueryResult {
 	readonly normalized: NormalizedAst;
 }
+
+export const NormalizedAstQueryDefinition = {
+	executor:        executeNormalizedAstQuery,
+	asciiSummarizer: (formatter, _processed, queryResults, result) => {
+		const out = queryResults as QueryResults<'normalized-ast'>['normalized-ast'];
+		result.push(`Query: ${bold('normalized-ast', formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
+		result.push(`   ╰ [Normalized AST](${normalizedAstToMermaidUrl(out.normalized.ast)})`);
+		return true;
+	},
+	schema: Joi.object({
+		type: Joi.string().valid('normalized-ast').required().description('The type of the query.'),
+	}).description('The normalized AST query simply returns the normalized AST, there is no need to pass it multiple times!')
+} as const satisfies SupportedQuery<'normalized-ast'>;

src/queries/catalog/static-slice-query/static-slice-query-format.ts

@@ -5,6 +5,12 @@ import type {
 	DEFAULT_SLICING_PIPELINE
 } from '../../../core/steps/pipeline/default-pipelines';
 import type { SlicingCriteria } from '../../../slicing/criterion/parse';
+import type { QueryResults, SupportedQuery } from '../../query';
+import { bold } from '../../../util/ansi';
+import { printAsMs } from '../../../util/time';
+import Joi from 'joi';
+import { executeStaticSliceClusterQuery } from './static-slice-query-executor';
+import { summarizeIdsIfTooLong } from '../../../documentation/doc-util/doc-query';
 
 /** Calculates and returns all clusters encountered in the dataflow graph. */
 export interface StaticSliceQuery extends BaseQueryFormat {
@@ -30,3 +36,34 @@ export interface StaticSliceQueryResult extends BaseQueryResult {
 		Omit<PipelineOutput<typeof DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE>, keyof PipelineOutput<typeof DEFAULT_DATAFLOW_PIPELINE>>
 	>
 }
+
+export const StaticSliceQueryDefinition = {
+	executor:        executeStaticSliceClusterQuery,
+	asciiSummarizer: (formatter, _processed, queryResults, result) => {
+		const out = queryResults as QueryResults<'static-slice'>['static-slice'];
+		result.push(`Query: ${bold('static-slice', formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
+		for(const [fingerprint, obj] of Object.entries(out.results)) {
+			const { criteria, noMagicComments, noReconstruction } = JSON.parse(fingerprint) as StaticSliceQuery;
+			const addons = [];
+			if(noReconstruction) {
+				addons.push('no reconstruction');
+			}
+			if(noMagicComments) {
+				addons.push('no magic comments');
+			}
+			result.push(`   ╰ Slice for {${criteria.join(', ')}} ${addons.join(', ')}`);
+			if('reconstruct' in obj) {
+				result.push('     ╰ Code (newline as <code>&#92;n</code>): <code>' + obj.reconstruct.code.split('\n').join('\\n') + '</code>');
+			} else {
+				result.push(`     ╰ Id List: {${summarizeIdsIfTooLong([...obj.slice.result])}}`);
+			}
+		}
+		return true;
+	},
+	schema: Joi.object({
+		type:             Joi.string().valid('static-slice').required().description('The type of the query.'),
+		criteria:         Joi.array().items(Joi.string()).min(0).required().description('The slicing criteria to use.'),
+		noReconstruction: Joi.boolean().optional().description('Do not reconstruct the slice into readable code.'),
+		noMagicComments:  Joi.boolean().optional().description('Should the magic comments (force-including lines within the slice) be ignored?')
+	}).description('Slice query used to slice the dataflow graph')
+} as const satisfies SupportedQuery<'static-slice'>;