[Query API] Dataflow Query (#1035)

src/cli/repl/commands/repl-query.ts

@@ -7,7 +7,7 @@ import { splitAtEscapeSensitive } from '../../../util/args';
 import type { OutputFormatter } from '../../../util/ansi';
 import { bold, italic } from '../../../util/ansi';
 
-import type { CallContextQuerySubKindResult } from '../../../queries/call-context-query/call-context-query-format';
+import type { CallContextQuerySubKindResult } from '../../../queries/catalog/call-context-query/call-context-query-format';
 import { describeSchema } from '../../../util/schema';
 import type { Query, QueryResults, SupportedQueryTypes } from '../../../queries/query';
 import { executeQueries } from '../../../queries/query';
@@ -17,6 +17,7 @@ import { jsonReplacer } from '../../../util/json';
 import { AnyQuerySchema, QueriesSchema } from '../../../queries/query-schema';
 import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id';
 import { BuiltIn } from '../../../dataflow/environments/built-in';
+import { graphToMermaidUrl } from '../../../util/mermaid/dfg';
 
 async function getDataflow(shell: RShell, remainingLine: string) {
 	return await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, {
@@ -126,6 +127,11 @@ export function asciiSummaryOfQueryResult(formatter: OutputFormatter, totalInMs:
 			result.push(`Query: ${bold(query, formatter)} (${out['.meta'].timing.toFixed(0)}ms)`);
 			result.push(asciiCallContext(formatter, out, processed));
 			continue;
+		} else if(query === 'dataflow') {
+			const out = queryResults as QueryResults<'dataflow'>['dataflow'];
+			result.push(`Query: ${bold(query, formatter)} (${out['.meta'].timing.toFixed(0)}ms)`);
+			result.push(`   ╰ [Dataflow Graph](${graphToMermaidUrl(out.graph)})`);
+			continue;
 		}
 
 		result.push(`Query: ${bold(query, formatter)}`);

src/documentation/data/server/doc-data-server-messages.ts

@@ -26,7 +26,7 @@ import {
 	responseQueryMessage
 } from '../../../cli/repl/server/messages/message-query';
 import { exampleQueryCode } from '../query/example-query-code';
-import { CallTargets } from '../../../queries/call-context-query/call-context-query-format';
+import { CallTargets } from '../../../queries/catalog/call-context-query/call-context-query-format';
 import { requestLineageMessage, responseLineageMessage } from '../../../cli/repl/server/messages/message-lineage';
 
 export function documentAllMessages() {

src/documentation/doc-util/doc-code.ts

@@ -1,3 +1,13 @@
+import { jsonReplacer } from '../../util/json';
+
 export function codeBlock(language: string, code: string | undefined): string {
 	return `\n\`\`\`${language}\n${code?.trim() ?? ''}\n\`\`\`\n`;
 }
+
+export function jsonWithLimit(object: object, maxLength: number = 5_000, tooLongText: string = '_As the code is pretty long, we inhibit pretty printing and syntax highlighting (JSON):_'): string {
+	const prettyPrinted = JSON.stringify(object, jsonReplacer, 2);
+	return `
+${prettyPrinted.length > maxLength ? tooLongText : ''}
+${codeBlock(prettyPrinted.length > maxLength ? 'text' : 'json', prettyPrinted.length > 5_000 ? JSON.stringify(object, jsonReplacer) : prettyPrinted)}
+`;
+}

src/documentation/doc-util/doc-query.ts

@@ -12,6 +12,7 @@ import { FlowrWikiBaseRef, getFilePathMd } from './doc-files';
 import type { SupportedVirtualQueryTypes } from '../../queries/virtual-query/virtual-queries';
 import type { VirtualCompoundConstraint } from '../../queries/virtual-query/compound-query';
 import { printDfGraphForCode } from './doc-dfg';
+import { jsonWithLimit } from './doc-code';
 
 export interface ShowQueryOptions {
 	readonly showCode?:       boolean;
@@ -34,8 +35,6 @@ export async function showQuery<
 The analysis required _${printAsMs(duration)}_ (including parsing and normalization and the query) within the generation environment.
 	`.trim();
 
-	const resultAsString = JSON.stringify(results, jsonReplacer, 2);
-
 	return `
 
 \`\`\`json
@@ -47,7 +46,7 @@ ${collapseResult ? ' <details> <summary style="color:gray">Show Results</summary
 _Results (prettified and summarized):_
 
 ${
-	asciiSummaryOfQueryResult(markdownFormatter, duration, results as QueryResults<'call-context'>, analysis)
+	asciiSummaryOfQueryResult(markdownFormatter, duration, results as QueryResults<SupportedQueryTypes>, analysis)
 }
 
 <details> <summary style="color:gray">Show Detailed Results as Json</summary>
@@ -57,9 +56,7 @@ ${metaInfo}
 In general, the JSON contains the Ids of the nodes in question as they are present in the normalized AST or the dataflow graph of flowR.
 Please consult the [Interface](${FlowrWikiBaseRef}/Interface) wiki page for more information on how to get those.
 
-\`\`\`json
-${resultAsString}
-\`\`\`
+${jsonWithLimit(results)}
 
 </details>
 

src/documentation/doc-util/doc-server-message.ts

@@ -5,7 +5,7 @@ import { markdownFormatter } from '../../util/ansi';
 import type { FlowrMessage, IdMessageBase, MessageDefinition } from '../../cli/repl/server/messages/all-messages';
 import type { FakeServer, FakeSocket } from '../../../test/functionality/_helper/net';
 import { withSocket } from '../../../test/functionality/_helper/net';
-import { codeBlock } from './doc-code';
+import { jsonWithLimit } from './doc-code';
 import { printAsMs } from './doc-ms';
 import { guard } from '../../util/assert';
 
@@ -64,7 +64,6 @@ export interface MessagePingPongDocumentationArguments {
 
 function explainMsg(msg: IdMessageBase, type: 'request' | 'response', desc = '', open = false): string {
 	const bold: (s: string) => string = open ? s => `<b>${s}</b>` : s => s;
-	const msgPrettyPrint = JSON.stringify(msg, null, 2);
 	return `
 <li> ${bold( '<code>' + msg.type + `</code> (${type})`)}
 <details${open ? ' open' : ''}> 
@@ -73,8 +72,7 @@ function explainMsg(msg: IdMessageBase, type: 'request' | 'response', desc = '',
 
 ${desc}
 
-${msgPrettyPrint.length > 5_000 ? '_As the message is pretty long, we inhibit pretty printing and syntax highlighting:_' : ''}
-${codeBlock(msgPrettyPrint.length > 5_000 ? 'text' : 'json', msgPrettyPrint.length > 5_000 ? JSON.stringify(msg) : msgPrettyPrint)}
+${jsonWithLimit(msg)}
 
 </details>
 </li>

src/documentation/print-dataflow-graph-wiki.ts

@@ -789,7 +789,9 @@ Additionally, you may be interested in the set of [Unknown Side Effects](#unknow
 > [!TIP]
 > If you want to investigate the dataflow graph,
 > you can either use the [Visual Studio Code extension](${FlowrGithubBaseRef}/vscode-flowr) or the ${getReplCommand('dataflow*')}
-> command in the REPL (see the [Interface wiki page](${FlowrWikiBaseRef}/Interface) for more information).
+> command in the REPL (see the [Interface wiki page](${FlowrWikiBaseRef}/Interface) for more information). When using _flowR_ as a library, you may use the functions in ${getFilePathMd('../util/mermaid/dfg.ts')}.
+> 
+> If you receive a dataflow graph in its serialized form (e.g., by talking to a [_flowR_ server](${FlowrWikiBaseRef}/Interface)), you can use \`${DataflowGraph.name}::${DataflowGraph.fromJson.name}\` to retrieve the graph from the JSON representation.
 
 ${await printDfGraphForCode(shell,'x <- 3\ny <- x + 1\ny')}
 

src/documentation/print-query-wiki.ts

@@ -5,24 +5,25 @@ import { LogLevel } from '../util/log';
 import { executeQueries } from '../queries/query';
 import { FlowrWikiBaseRef, getFilePathMd } from './doc-util/doc-files';
 import { explainQueries, registerQueryDocumentation, showQuery, tocForQueryType } from './doc-util/doc-query';
-import { CallTargets } from '../queries/call-context-query/call-context-query-format';
+import { CallTargets } from '../queries/catalog/call-context-query/call-context-query-format';
 import { describeSchema } from '../util/schema';
 import { QueriesSchema } from '../queries/query-schema';
 import { markdownFormatter } from '../util/ansi';
-import { executeCallContextQueries } from '../queries/call-context-query/call-context-query-executor';
+import { executeCallContextQueries } from '../queries/catalog/call-context-query/call-context-query-executor';
 import { executeCompoundQueries } from '../queries/virtual-query/compound-query';
 import { autoGenHeader } from './doc-util/doc-auto-gen';
 import { exampleQueryCode } from './data/query/example-query-code';
 import { details } from './doc-util/doc-structure';
 import { codeBlock } from './doc-util/doc-code';
+import { executeDataflowQuery } from '../queries/catalog/dataflow-query/dataflow-query-executor';
 
 
 registerQueryDocumentation('call-context', {
 	name:             'Call-Context Query',
 	type:             'active',
 	shortDescription: 'Finds all calls in a set of files that matches specified criteria.',
 	functionName:     executeCallContextQueries.name,
-	functionFile:     '../queries/call-context-query/call-context-query-executor.ts',
+	functionFile:     '../queries/catalog/call-context-query/call-context-query-executor.ts',
 	buildExplanation: async(shell: RShell) => {
 		return `
 Call context queries may be used to identify calls to specific functions that match criteria of your interest.
@@ -83,6 +84,27 @@ my_test_function()
 		`;
 	}
 });
+registerQueryDocumentation('dataflow', {
+	name:             'Dataflow Query',
+	type:             'active',
+	shortDescription: 'Returns the dataflow graph of the given code.',
+	functionName:     executeDataflowQuery.name,
+	functionFile:     '../queries/catalog/dataflow-query/dataflow-query-executor.ts',
+	buildExplanation: async(shell: RShell) => {
+		return `
+Maybe you want to handle only the result of the query execution, or you just need the [dataflow graph](${FlowrWikiBaseRef}/Dataflow%20Graph) again.
+This query type does exactly that!
+
+Using the example code from above, the following query returns the dataflow graph of the code:
+${
+	await showQuery(shell, exampleQueryCode, [{
+		type: 'dataflow'
+	}], { showCode: true })
+}
+		`;
+	}
+});
+
 registerQueryDocumentation('compound', {
 	name:             'Compound Query',
 	type:             'virtual',
@@ -122,7 +144,7 @@ ${
 }
 
 However, compound queries become more useful whenever common arguments can not be expressed as a union in one of their properties.
-Additionally, you can still overwrite default arguments. 
+Additionally, you can still overwrite default arguments.
 In the following, we (by default) want all calls to not resolve to a local definition, except for those to \`print\` for which we explicitly
 want to resolve to a local definition:
 
@@ -138,12 +160,13 @@ ${
 	}], { showCode: false })
 }
 
-Now, the results no longer contain calls to \`plot\` that are not defined locally. 
+Now, the results no longer contain calls to \`plot\` that are not defined locally.
 
 		`;
 	}
 });
 
+
 async function getText(shell: RShell) {
 	const rversion = (await shell.usedRVersion())?.format() ?? 'unknown';
 	return `${autoGenHeader({ filename: module.filename, purpose: 'query API', rVersion: rversion })}

src/queries/call-context-query/call-context-query-executor.ts → src/queries/catalog/call-context-query/call-context-query-executor.ts

@@ -1,4 +1,4 @@
-import type { DataflowGraph } from '../../dataflow/graph/graph';
+import type { DataflowGraph } from '../../../dataflow/graph/graph';
 import type {
 	CallContextQuery,
 	CallContextQueryKindResult,
@@ -7,20 +7,20 @@ import type {
 	SubCallContextQueryFormat
 } from './call-context-query-format';
 import { CallTargets } from './call-context-query-format';
-import type { NodeId } from '../../r-bridge/lang-4.x/ast/model/processing/node-id';
-import { recoverContent  } from '../../r-bridge/lang-4.x/ast/model/processing/node-id';
-import { VertexType } from '../../dataflow/graph/vertex';
-import { assertUnreachable } from '../../util/assert';
-import { edgeIncludesType, EdgeType } from '../../dataflow/graph/edge';
-import { resolveByName } from '../../dataflow/environments/resolve-by-name';
-import { BuiltIn } from '../../dataflow/environments/built-in';
-import type { ControlFlowGraph } from '../../util/cfg/cfg';
-import { extractCFG } from '../../util/cfg/cfg';
-import { TwoLayerCollector } from '../two-layer-collector';
-import type { BasicQueryData } from '../query';
-import { compactRecord } from '../../util/objects';
-import { visitInReverseOrder } from '../../util/cfg/visitor';
-import { ReferenceType } from '../../dataflow/environments/identifier';
+import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id';
+import { recoverContent  } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id';
+import { VertexType } from '../../../dataflow/graph/vertex';
+import { assertUnreachable } from '../../../util/assert';
+import { edgeIncludesType, EdgeType } from '../../../dataflow/graph/edge';
+import { resolveByName } from '../../../dataflow/environments/resolve-by-name';
+import { BuiltIn } from '../../../dataflow/environments/built-in';
+import type { ControlFlowGraph } from '../../../util/cfg/cfg';
+import { extractCFG } from '../../../util/cfg/cfg';
+import { TwoLayerCollector } from '../../two-layer-collector';
+import type { BasicQueryData } from '../../query';
+import { compactRecord } from '../../../util/objects';
+import { visitInReverseOrder } from '../../../util/cfg/visitor';
+import { ReferenceType } from '../../../dataflow/environments/identifier';
 
 function satisfiesCallTargets(id: NodeId, graph: DataflowGraph, callTarget: CallTargets): NodeId[] | 'no'  {
 	const callVertex = graph.get(id);
@@ -211,9 +211,10 @@ function retrieveAllCallAliases(nodeId: NodeId, graph: DataflowGraph): Map<strin
  * Multi-stage call context query resolve.
  *
  * 1. Resolve all calls in the DF graph that match the respective {@link DefaultCallContextQueryFormat#callName} regex.
- * 2. Identify their respective call targets, if {@link DefaultCallContextQueryFormat#callTargets} is set to be non-any.
+ * 2. If there is an alias attached, consider all call traces.
+ * 3. Identify their respective call targets, if {@link DefaultCallContextQueryFormat#callTargets} is set to be non-any.
  *    This happens during the main resolution!
- * 3. Attach `linkTo` calls to the respective calls.
+ * 4. Attach `linkTo` calls to the respective calls.
  */
 export function executeCallContextQueries({ graph, ast }: BasicQueryData, queries: readonly CallContextQuery[]): CallContextQueryResult {
 	/* omit performance page load */

src/queries/call-context-query/call-context-query-format.ts → src/queries/catalog/call-context-query/call-context-query-format.ts

@@ -1,5 +1,5 @@
-import type { BaseQueryFormat, BaseQueryResult } from '../base-query-format';
-import type { NodeId } from '../../r-bridge/lang-4.x/ast/model/processing/node-id';
+import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
+import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id';
 
 export enum CallTargets {
 	/** call targets a function that is not defined locally (e.g., the call targets a library function) */

src/queries/catalog/dataflow-query/dataflow-query-executor.ts

@@ -0,0 +1,17 @@
+import type { BasicQueryData } from '../../query';
+import type { DataflowQuery, DataflowQueryResult } from './dataflow-query-format';
+import { log } from '../../../util/log';
+
+
+export function executeDataflowQuery({ graph }: BasicQueryData, queries: readonly DataflowQuery[]): DataflowQueryResult {
+	if(queries.length !== 1) {
+		log.warn('Dataflow query expects only up to one query, but got', queries.length);
+	}
+	return {
+		'.meta': {
+			/* there is no sense in measuring a get */
+			timing: 0
+		},
+		graph
+	};
+}

src/queries/catalog/dataflow-query/dataflow-query-format.ts

@@ -0,0 +1,14 @@
+import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
+import type { DataflowGraph } from '../../../dataflow/graph/graph';
+
+/**
+ * Simple re-returns the dataflow graph of the analysis.
+ */
+export interface DataflowQuery extends BaseQueryFormat {
+	readonly type: 'dataflow';
+}
+
+export interface DataflowQueryResult extends BaseQueryResult {
+	/** Please be aware that this is the graph in its JSON representation, use {@link DataflowGraph#fromJson} if the result is serialized */
+	readonly graph: DataflowGraph;
+}

src/queries/query-schema.ts

@@ -1,5 +1,5 @@
 import Joi from 'joi';
-import { CallTargets } from './call-context-query/call-context-query-format';
+import { CallTargets } from './catalog/call-context-query/call-context-query-format';
 
 export const CallContextQuerySchema = Joi.object({
 	type:           Joi.string().valid('call-context').required().description('The type of the query.'),
@@ -14,8 +14,13 @@ export const CallContextQuerySchema = Joi.object({
 	}).optional().description('Links the current call to the last call of the given kind. This way, you can link a call like `points` to the latest graphics plot etc.')
 }).description('Call context query used to find calls in the dataflow graph');
 
+export const DataflowQuerySchema = Joi.object({
+	type: Joi.string().valid('dataflow').required().description('The type of the query.'),
+}).description('The dataflow query simply returns the dataflow graph, there is no need to pass it multiple times!');
+
 export const SupportedQueriesSchema = Joi.alternatives(
-	CallContextQuerySchema
+	CallContextQuerySchema,
+	DataflowQuerySchema
 ).description('Supported queries');
 
 export const CompoundQuerySchema = Joi.object({

src/queries/query.ts

@@ -1,15 +1,17 @@
-import type { CallContextQuery } from './call-context-query/call-context-query-format';
+import type { CallContextQuery } from './catalog/call-context-query/call-context-query-format';
 import type { DataflowGraph } from '../dataflow/graph/graph';
 import type { BaseQueryFormat, BaseQueryResult } from './base-query-format';
-import { executeCallContextQueries } from './call-context-query/call-context-query-executor';
+import { executeCallContextQueries } from './catalog/call-context-query/call-context-query-executor';
 import { guard } from '../util/assert';
 import type { VirtualQueryArgumentsWithType } from './virtual-query/virtual-queries';
 import { SupportedVirtualQueries } from './virtual-query/virtual-queries';
 import type { Writable } from 'ts-essentials';
 import type { VirtualCompoundConstraint } from './virtual-query/compound-query';
 import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate';
+import { executeDataflowQuery } from './catalog/dataflow-query/dataflow-query-executor';
+import type { DataflowQuery } from './catalog/dataflow-query/dataflow-query-format';
 
-export type Query = CallContextQuery;
+export type Query = CallContextQuery | DataflowQuery;
 
 export type QueryArgumentsWithType<QueryType extends BaseQueryFormat['type']> = Query & { type: QueryType };
 
@@ -26,7 +28,8 @@ type SupportedQueries = {
 }
 
 export const SupportedQueries = {
-	'call-context': executeCallContextQueries
+	'call-context': executeCallContextQueries,
+	'dataflow':     executeDataflowQuery
 } as const satisfies SupportedQueries;
 
 export type SupportedQueryTypes = keyof typeof SupportedQueries;
@@ -38,7 +41,7 @@ export function executeQueriesOfSameType<SpecificQuery extends Query>(data: Basi
 	guard(queries.every(q => q.type === queries[0].type), 'All queries must have the same type');
 	const executor = SupportedQueries[queries[0].type];
 	guard(executor !== undefined, `Unsupported query type: ${queries[0].type}`);
-	return executor(data, queries) as QueryResult<SpecificQuery['type']>;
+	return executor(data, queries as never) as QueryResult<SpecificQuery['type']>;
 }
 
 function isVirtualQuery<
@@ -67,7 +70,7 @@ function groupQueriesByType<
 				addQuery(subQuery);
 			}
 		} else {
-			addQuery(query as Query);
+			addQuery(query);
 		}
 	}
 	return grouped;
@@ -80,7 +83,7 @@ export type QueryResults<Base extends SupportedQueryTypes> = {
 
 
 type OmitFromValues<T, K extends string | number | symbol> = {
-	[P in keyof T]: Omit<T[P], K>
+	[P in keyof T]?: Omit<T[P], K>
 }
 
 export type QueryResultsWithoutMeta<Queries extends Query> = OmitFromValues<Omit<QueryResults<Queries['type']>, '.meta'>, '.meta'>;