[Query API] Slice Query (#1073)

* feat(slice-query): base slice query implementation

* doc(slice-query): document the slice query

src/cli/repl/commands/repl-query.ts

@@ -22,6 +22,7 @@ import { normalizedAstToMermaidUrl } from '../../../util/mermaid/ast';
 
 import { printAsMs } from '../../../util/time';
 import { textWithTooltip } from '../../../documentation/doc-util/doc-hover-over';
+import type { StaticSliceQuery } from '../../../queries/catalog/static-slice-query/static-slice-query-format';
 
 async function getDataflow(shell: RShell, remainingLine: string) {
 	return await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, {
@@ -162,6 +163,26 @@ export function asciiSummaryOfQueryResult(formatter: OutputFormatter, totalInMs:
 			result.push(`Query: ${bold(query, formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
 			result.push(`   ╰ [Normalized AST](${normalizedAstToMermaidUrl(out.normalized.ast)})`);
 			continue;
+		} else if(query === 'static-slice') {
+			const out = queryResults as QueryResults<'static-slice'>['static-slice'];
+			result.push(`Query: ${bold(query, formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
+			for(const [fingerprint, obj] of Object.entries(out.results)) {
+				const { criteria, noMagicComments, noReconstruction } = JSON.parse(fingerprint) as StaticSliceQuery;
+				const addons = [];
+				if(noReconstruction) {
+					addons.push('no reconstruction');
+				}
+				if(noMagicComments) {
+					addons.push('no magic comments');
+				}
+				result.push(`   ╰ Slice for {${criteria.join(', ')}} ${addons.join(', ')}`);
+				if('reconstruct' in obj) {
+					result.push('     ╰ Code (newline as <code>&#92;n</code>): <code>' + obj.reconstruct.code.split('\n').join('\\n') + '</code>');
+				} else {
+					result.push(`     ╰ Id List: {${summarizeIdsIfTooLong([...obj.slice.result])}}`);
+				}
+			}
+			continue;
 		} else if(query === 'dataflow-cluster') {
 			const out = queryResults as QueryResults<'dataflow-cluster'>['dataflow-cluster'];
 			result.push(`Query: ${bold(query, formatter)} (${out['.meta'].timing.toFixed(0)}ms)`);

src/core/steps/pipeline/default-pipelines.ts

@@ -10,6 +10,7 @@ import { NAIVE_RECONSTRUCT } from '../all/static-slicing/10-reconstruct';
 
 export const DEFAULT_SLICING_PIPELINE = createPipeline(PARSE_WITH_R_SHELL_STEP, NORMALIZE, STATIC_DATAFLOW, STATIC_SLICE, NAIVE_RECONSTRUCT);
 export const DEFAULT_SLICE_AND_RECONSTRUCT_PIPELINE = DEFAULT_SLICING_PIPELINE;
+export const DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE = createPipeline(PARSE_WITH_R_SHELL_STEP, NORMALIZE, STATIC_DATAFLOW, STATIC_SLICE);
 
 export const DEFAULT_DATAFLOW_PIPELINE = createPipeline(PARSE_WITH_R_SHELL_STEP, NORMALIZE, STATIC_DATAFLOW);
 

src/documentation/data/server/doc-data-server-messages.ts

@@ -28,8 +28,9 @@ import {
 import { exampleQueryCode } from '../query/example-query-code';
 import { CallTargets } from '../../../queries/catalog/call-context-query/call-context-query-format';
 import { requestLineageMessage, responseLineageMessage } from '../../../cli/repl/server/messages/message-lineage';
+import { block } from '../../doc-util/doc-structure';
 
-export function documentAllMessages() {
+export function documentAllServerMessages() {
 
 	documentServerMessage({
 		title:                  'Hello',
@@ -244,9 +245,14 @@ While the context is derived from the \`filename\`, we currently offer no way to
     end
     deactivate  Server
 	`,
-		shortDescription: 'The server slices a file based on the given criteria.',
+		shortDescription: `([DEPRECATED](${FlowrWikiBaseRef}/Query%20API)) The server slices a file based on the given criteria.`,
 		text:             async(shell: RShell) => {
 			return `
+${block({
+		type:    'WARNING',
+		content: `We deprecated the slice request in favor of the \`static-slice\` [Query](${FlowrWikiBaseRef}/Query%20API).`
+	})}
+
 To slice, you have to send a file analysis request first. The \`filetoken\` you assign is of use here as you can re-use it to repeatedly slice the same file.
 Besides that, you only need to add an array of slicing criteria, using one of the formats described on the [terminology wiki page](${FlowrWikiBaseRef}/Terminology#slicing-criterion) 
 (however, instead of using \`;\`, you can simply pass separate array elements).

src/documentation/print-interface-wiki.ts

@@ -5,7 +5,7 @@ import { FlowrGithubBaseRef, FlowrNpmRef, FlowrWikiBaseRef, getFileContentFromRo
 import { autoGenHeader } from './doc-util/doc-auto-gen';
 import { getCliLongOptionOf, getReplCommand, multipleCliOptions } from './doc-util/doc-cli-option';
 import { printServerMessages } from './doc-util/doc-server-message';
-import { documentAllMessages } from './data/server/doc-data-server-messages';
+import { documentAllServerMessages } from './data/server/doc-data-server-messages';
 import { codeBlock } from './doc-util/doc-code';
 import type { FileAnalysisRequestMessage } from '../cli/repl/server/messages/message-analysis';
 import { fileProtocol, removeRQuotes, requestFromInput } from '../r-bridge/retriever';
@@ -21,7 +21,7 @@ import { NewIssueUrl } from './doc-util/doc-issue';
 import { PipelineExecutor } from '../core/pipeline-executor';
 
 async function explainServer(shell: RShell): Promise<string> {
-	documentAllMessages();
+	documentAllServerMessages();
 
 	return `
 As explained in the [Overview](${FlowrWikiBaseRef}/Overview), you can simply run the [TCP](https://de.wikipedia.org/wiki/Transmission_Control_Protocol)&nbsp;server by adding the ${getCliLongOptionOf('flowr', 'server', true)} flag (and, due to the interactive mode, exit with the conventional <kbd>CTRL</kbd>+<kbd>C</kbd>).

src/documentation/print-query-wiki.ts

@@ -19,6 +19,7 @@ import { executeDataflowQuery } from '../queries/catalog/dataflow-query/dataflow
 import { executeIdMapQuery } from '../queries/catalog/id-map-query/id-map-query-executor';
 import { executeNormalizedAstQuery } from '../queries/catalog/normalized-ast-query/normalized-ast-query-executor';
 import { executeDataflowClusterQuery } from '../queries/catalog/cluster-query/cluster-query-executor';
+import { executeStaticSliceClusterQuery } from '../queries/catalog/static-slice-query/static-slice-query-executor';
 
 
 registerQueryDocumentation('call-context', {
@@ -246,6 +247,51 @@ Now, the results no longer contain calls to \`plot\` that are not defined locall
 });
 
 
+registerQueryDocumentation('static-slice', {
+	name:             'Static Slice Query',
+	type:             'active',
+	shortDescription: 'Slice the dataflow graph reducing the code to just the parts relevant for the given criteria.',
+	functionName:     executeStaticSliceClusterQuery.name,
+	functionFile:     '../queries/catalog/static-slice-query/static-slice-query-executor.ts',
+	buildExplanation: async(shell: RShell) => {
+		const exampleCode = 'x <- 1\ny <- 2\nx';
+		return `
+To slice, _flowR_ needs one thing from you: a variable or a list of variables (function calls are supported to, referring to the anonymous
+return of the call) that you want to slice the dataflow graph for. 
+Given this, the slice is essentially the subpart of the program that may influence the value of the variables you are interested in.
+To specify a variable of interest, you have to present flowR with a [slicing criterion](${FlowrWikiBaseRef}/Terminology#slicing-criterion) (or, respectively, an array of them).
+
+To exemplify the capabilities, consider the following code:
+${codeBlock('r', exampleCode)}
+If you are interested in the parts required for the use of \`x\` in the last line, you can use the following query:
+
+${
+	await showQuery(shell, exampleCode, [{
+		type:     'static-slice',
+		criteria: ['3@x']
+	}], { showCode: false })
+}
+
+In general you may be uninterested in seeing the reconstructed version and want to save some computation time, for this,
+you can use the \`noReconstruction\` flag.
+
+${
+	details('No Reconstruction Example',
+		await showQuery(shell, exampleCode, [{
+			type:             'static-slice',
+			criteria:         ['3@x'],
+			noReconstruction: true
+		}], { showCode: false })
+	)
+}
+
+You can disable [magic comments](${FlowrWikiBaseRef}/Interface#slice-magic-comments) using the \`noMagicComments\` flag.
+This query replaces the old [\`request-slice\`](${FlowrWikiBaseRef}/Interface#message-request-slice) message.
+		`;
+	}
+});
+
+
 async function getText(shell: RShell) {
 	const rversion = (await shell.usedRVersion())?.format() ?? 'unknown';
 	return `${autoGenHeader({ filename: module.filename, purpose: 'query API', rVersion: rversion })}

src/queries/catalog/call-context-query/call-context-query-executor.ts

@@ -46,7 +46,7 @@ function satisfiesCallTargets(id: NodeId, graph: DataflowGraph, callTarget: Call
 		 * including any potential built-in mapping.
 		 */
 		const reResolved = resolveByName(info.name, info.environment, ReferenceType.Unknown);
-		if(reResolved && reResolved.some(t => t.definedAt === BuiltIn)) {
+		if(reResolved?.some(t => t.definedAt === BuiltIn)) {
 			builtIn = true;
 		}
 	}

src/queries/catalog/static-slice-query/static-slice-query-executor.ts

@@ -0,0 +1,38 @@
+import type { BasicQueryData } from '../../query';
+import type { StaticSliceQuery, StaticSliceQueryResult } from './static-slice-query-format';
+import { staticSlicing } from '../../../slicing/static/static-slicer';
+import { reconstructToCode } from '../../../reconstruct/reconstruct';
+import { doNotAutoSelect } from '../../../reconstruct/auto-select/auto-select-defaults';
+import { makeMagicCommentHandler } from '../../../reconstruct/auto-select/magic-comments';
+import { log } from '../../../util/log';
+
+export function fingerPrintOfQuery(query: StaticSliceQuery): string {
+	return JSON.stringify(query);
+}
+
+export function executeStaticSliceClusterQuery({ graph, ast }: BasicQueryData, queries: readonly StaticSliceQuery[]): StaticSliceQueryResult {
+	const start = Date.now();
+	const results: StaticSliceQueryResult['results'] = {};
+	for(const query of queries) {
+		const key = fingerPrintOfQuery(query);
+		if(results[key]) {
+			log.warn(`Duplicate Key for slicing-query: ${key}, skipping...`);
+		}
+		const { criteria, noReconstruction, noMagicComments } = query;
+		const slice = staticSlicing(graph, ast, criteria);
+		if(noReconstruction) {
+			results[key] = { slice };
+		} else {
+			results[key] = {
+				slice,
+				reconstruct: reconstructToCode(ast, slice.result, noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect))
+			};
+		}
+	}
+	return {
+		'.meta': {
+			timing: Date.now() - start
+		},
+		results
+	};
+}

src/queries/catalog/static-slice-query/static-slice-query-format.ts

@@ -0,0 +1,32 @@
+import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
+import type { PipelineOutput } from '../../../core/steps/pipeline/pipeline';
+import type {
+	DEFAULT_DATAFLOW_PIPELINE, DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE,
+	DEFAULT_SLICING_PIPELINE
+} from '../../../core/steps/pipeline/default-pipelines';
+import type { SlicingCriteria } from '../../../slicing/criterion/parse';
+
+/** Calculates and returns all clusters encountered in the dataflow graph. */
+export interface StaticSliceQuery extends BaseQueryFormat {
+	readonly type:              'static-slice';
+	/** The slicing criteria to use */
+	readonly criteria:          SlicingCriteria,
+	/** do not reconstruct the slice into readable code */
+	readonly noReconstruction?: boolean;
+	/** Should the magic comments (force-including lines within the slice) be ignored? */
+	readonly noMagicComments?:  boolean
+}
+
+export interface StaticSliceQueryResult extends BaseQueryResult {
+	/**
+	 * only contains the results of the slice steps to not repeat ourselves, this does not contain the reconstruction
+	 * if you set the {@link SliceQuery#noReconstruction|noReconstruction} flag.
+	 *
+	 * The keys are serialized versions of the used queries (i.e., the result of `JSON.stringify`).
+	 * This implies that multiple slice queries with the same query configuration will _not_ be re-executed.
+	 */
+	results: Record<string,
+		Omit<PipelineOutput<typeof DEFAULT_SLICING_PIPELINE>, keyof PipelineOutput<typeof DEFAULT_DATAFLOW_PIPELINE>> |
+		Omit<PipelineOutput<typeof DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE>, keyof PipelineOutput<typeof DEFAULT_DATAFLOW_PIPELINE>>
+	>
+}

src/queries/query-schema.ts

@@ -31,13 +31,21 @@ export const DataflowClusterQuerySchema = Joi.object({
 	type: Joi.string().valid('dataflow-cluster').required().description('The type of the query.'),
 }).description('The cluster query calculates and returns all clusters in the dataflow graph.');
 
+export const StaticSliceQuerySchema = Joi.object({
+	type:             Joi.string().valid('static-slice').required().description('The type of the query.'),
+	criteria:         Joi.array().items(Joi.string()).min(0).required().description('The slicing criteria to use.'),
+	noReconstruction: Joi.boolean().optional().description('Do not reconstruct the slice into readable code.'),
+	noMagicComments:  Joi.boolean().optional().description('Should the magic comments (force-including lines within the slice) be ignored?')
+}).description('Slice query used to slice the dataflow graph');
+
 
 export const SupportedQueriesSchema = Joi.alternatives(
 	CallContextQuerySchema,
 	DataflowQuerySchema,
 	IdMapQuerySchema,
 	NormalizedAstQuerySchema,
-	DataflowClusterQuerySchema
+	DataflowClusterQuerySchema,
+	StaticSliceQuerySchema
 ).description('Supported queries');
 
 export const CompoundQuerySchema = Joi.object({

src/queries/query.ts

@@ -16,8 +16,10 @@ import { executeNormalizedAstQuery } from './catalog/normalized-ast-query/normal
 import type {	NormalizedAstQuery } from './catalog/normalized-ast-query/normalized-ast-query-format';
 import type { DataflowClusterQuery } from './catalog/cluster-query/cluster-query-format';
 import { executeDataflowClusterQuery } from './catalog/cluster-query/cluster-query-executor';
+import type { StaticSliceQuery } from './catalog/static-slice-query/static-slice-query-format';
+import { executeStaticSliceClusterQuery } from './catalog/static-slice-query/static-slice-query-executor';
 
-export type Query = CallContextQuery | DataflowQuery | NormalizedAstQuery | IdMapQuery | DataflowClusterQuery;
+export type Query = CallContextQuery | DataflowQuery | NormalizedAstQuery | IdMapQuery | DataflowClusterQuery | StaticSliceQuery;
 
 export type QueryArgumentsWithType<QueryType extends BaseQueryFormat['type']> = Query & { type: QueryType };
 
@@ -39,6 +41,7 @@ export const SupportedQueries = {
 	'id-map':           executeIdMapQuery,
 	'normalized-ast':   executeNormalizedAstQuery,
 	'dataflow-cluster': executeDataflowClusterQuery,
+	'static-slice':     executeStaticSliceClusterQuery
 } as const satisfies SupportedQueries;
 
 export type SupportedQueryTypes = keyof typeof SupportedQueries;

test/functionality/_helper/query.ts

@@ -47,8 +47,12 @@ function normalizeResults<Queries extends Query>(result: QueryResults<Queries['t
 export function assertQuery<
 	Queries extends Query,
 	VirtualArguments extends VirtualCompoundConstraint<Queries['type']> = VirtualCompoundConstraint<Queries['type']>
->(name: string | TestLabel, shell: RShell, code: string, queries: readonly (Queries | VirtualQueryArgumentsWithType<Queries['type'], VirtualArguments>)[], expected:
-	QueryResultsWithoutMeta<Queries> | ((info: PipelineOutput<typeof DEFAULT_DATAFLOW_PIPELINE>) => QueryResultsWithoutMeta<Queries>)
+>(
+	name: string | TestLabel,
+	shell: RShell,
+	code: string,
+	queries: readonly (Queries | VirtualQueryArgumentsWithType<Queries['type'], VirtualArguments>)[],
+	expected: QueryResultsWithoutMeta<Queries> | ((info: PipelineOutput<typeof DEFAULT_DATAFLOW_PIPELINE>) => (QueryResultsWithoutMeta<Queries> | Promise<QueryResultsWithoutMeta<Queries>>))
 ) {
 	const effectiveName = decorateLabelContext(name, ['query']);
 
@@ -68,7 +72,7 @@ export function assertQuery<
 		/* expect them to be deeply equal */
 		try {
 			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-			const expectedNormalized = typeof expected === 'function' ? expected(info) : expected;
+			const expectedNormalized = typeof expected === 'function' ? await expected(info) : expected;
 			assert.deepStrictEqual(normalized, expectedNormalized, 'The result of the call context query does not match the expected result');
 		} catch(e: unknown) {
 			console.error('Dataflow-Graph', dataflowGraphToMermaidUrl(info.dataflow));

test/functionality/dataflow/query/slice-query-tests.ts

@@ -0,0 +1,51 @@
+import { assertQuery } from '../../_helper/query';
+import { label } from '../../_helper/label';
+import { withShell } from '../../_helper/shell';
+import type {
+	StaticSliceQuery,
+	StaticSliceQueryResult
+} from '../../../../src/queries/catalog/static-slice-query/static-slice-query-format';
+import { fingerPrintOfQuery } from '../../../../src/queries/catalog/static-slice-query/static-slice-query-executor';
+import { PipelineExecutor } from '../../../../src/core/pipeline-executor';
+import {
+	DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE,
+	DEFAULT_SLICING_PIPELINE
+} from '../../../../src/core/steps/pipeline/default-pipelines';
+import { requestFromInput } from '../../../../src/r-bridge/retriever';
+import { doNotAutoSelect } from '../../../../src/reconstruct/auto-select/auto-select-defaults';
+import { makeMagicCommentHandler } from '../../../../src/reconstruct/auto-select/magic-comments';
+
+describe('Static Slice Query', withShell(shell => {
+	function testQuery(name: string, code: string, queries: readonly StaticSliceQuery[]) {
+		assertQuery(label(name), shell, code, queries, async() => {
+			const results: StaticSliceQueryResult['results'] = {};
+			for(const query of queries) {
+				const out = await new PipelineExecutor(query.noReconstruction ?  DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE : DEFAULT_SLICING_PIPELINE, {
+					shell:        shell,
+					request:      requestFromInput(code),
+					criterion:    query.criteria,
+					autoSelectIf: query.noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)
+				}).allRemainingSteps();
+				const key = fingerPrintOfQuery(query);
+				results[key] = query.noReconstruction ? { slice: out.slice } : { slice: out.slice, reconstruct: out.reconstruct };
+			}
+
+			return {
+				'static-slice': {
+					results
+				}
+			};
+		});
+	}
+
+	const baseQuery: StaticSliceQuery = { type: 'static-slice', criteria: ['1@x'] };
+	describe('With Reconstruction', () => {
+		testQuery('Single Expression', 'x + 1', [baseQuery]);
+		testQuery('Multiple Queries', 'x + 1', [baseQuery, baseQuery, baseQuery]);
+	});
+	const noReconstructQuery: StaticSliceQuery = { type: 'static-slice', criteria: ['1@x'], noReconstruction: true };
+	describe('Without Reconstruction', () => {
+		testQuery('Single Expression (No Reconstruct)', 'x + 1', [noReconstructQuery]);
+		testQuery('Multiple Queries (No Reconstruct)', 'x + 1', [noReconstructQuery, noReconstructQuery, noReconstructQuery]);
+	});
+}));