Integrate DFG information into CFG (#1254)

* refactor: quote github env variable

* doc(extract-cfg): document use of normalized ast

* feat(wip): insert flow edges

* feat(points-to): support function call resolution

.github/workflows/scripts/global-configuration.sh

@@ -7,7 +7,7 @@ fi
 
 function make_var {
    export $1=${2:-}
-   echo "$1=${2:-}" >> $GITHUB_ENV
+   echo "$1=${2:-}" >> "$GITHUB_ENV"
 }
 
 make_var ACTION_NODE_VERSION "22.13.x"

src/cli/repl/commands/repl-cfg.ts

@@ -1,26 +1,41 @@
-import type { ReplCommand } from './repl-main';
+import type { ReplCommand, ReplOutput } from './repl-main';
 import { extractCFG } from '../../../util/cfg/cfg';
-import { createNormalizePipeline } from '../../../core/steps/pipeline/default-pipelines';
+import { createDataflowPipeline } from '../../../core/steps/pipeline/default-pipelines';
 import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever';
 import { cfgToMermaid, cfgToMermaidUrl } from '../../../util/mermaid/cfg';
 import type { KnownParser } from '../../../r-bridge/parser';
+import { ColorEffect, Colors, FontStyles } from '../../../util/ansi';
+import clipboard from 'clipboardy';
 
 async function controlflow(parser: KnownParser, remainingLine: string) {
-	return await createNormalizePipeline(parser, {
+	return await createDataflowPipeline(parser, {
 		request: requestFromInput(remainingLine.trim())
 	}).allRemainingSteps();
 }
 
+function handleString(code: string): string {
+	return code.startsWith('"') ? JSON.parse(code) as string : code;
+}
+
+function formatInfo(out: ReplOutput, type: string): string {
+	return out.formatter.format(`Copied ${type} to clipboard.`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic });
+}
+
 export const controlflowCommand: ReplCommand = {
 	description:  `Get mermaid code for the control-flow graph of R code, start with '${fileProtocol}' to indicate a file`,
 	usageExample: ':controlflow',
 	aliases:      [ 'cfg', 'cf' ],
 	script:       false,
 	fn:           async(output, shell, remainingLine) => {
-		const result = await controlflow(shell, remainingLine);
+		const result = await controlflow(shell, handleString(remainingLine));
 
-		const cfg = extractCFG(result.normalize);
-		output.stdout(cfgToMermaid(cfg, result.normalize));
+		const cfg = extractCFG(result.normalize, result.dataflow.graph);
+		const mermaid = cfgToMermaid(cfg, result.normalize);
+		output.stdout(mermaid);
+		try {
+			clipboard.writeSync(mermaid);
+			output.stdout(formatInfo(output, 'mermaid code'));
+		} catch(e) { /* do nothing this is a service thing */ }
 	}
 };
 
@@ -30,9 +45,14 @@ export const controlflowStarCommand: ReplCommand = {
 	aliases:      [ 'cfg*', 'cf*' ],
 	script:       false,
 	fn:           async(output, shell, remainingLine) => {
-		const result = await controlflow(shell, remainingLine);
+		const result = await controlflow(shell, handleString(remainingLine));
 
-		const cfg = extractCFG(result.normalize);
-		output.stdout(cfgToMermaidUrl(cfg, result.normalize));
+		const cfg = extractCFG(result.normalize, result.dataflow.graph);
+		const mermaid = cfgToMermaidUrl(cfg, result.normalize);
+		output.stdout(mermaid);
+		try {
+			clipboard.writeSync(mermaid);
+			output.stdout(formatInfo(output, 'mermaid url'));
+		} catch(e) { /* do nothing this is a service thing */ }
 	}
 };

src/cli/repl/commands/repl-dataflow.ts

@@ -16,8 +16,8 @@ function handleString(code: string): string {
 	return code.startsWith('"') ? JSON.parse(code) as string : code;
 }
 
-function formatInfo(out: ReplOutput, type: string): string {
-	return out.formatter.format(`Copied ${type} to clipboard.`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic });
+function formatInfo(out: ReplOutput, type: string, timing: number): string {
+	return out.formatter.format(`Copied ${type} to clipboard (dataflow: ${timing}ms).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic });
 }
 
 export const dataflowCommand: ReplCommand = {
@@ -31,7 +31,7 @@ export const dataflowCommand: ReplCommand = {
 		output.stdout(mermaid);
 		try {
 			clipboard.writeSync(mermaid);
-			output.stdout(formatInfo(output, 'mermaid code'));
+			output.stdout(formatInfo(output, 'mermaid code', result.dataflow['.meta'].timing));
 		} catch(e) { /* do nothing this is a service thing */ }
 	}
 };
@@ -47,7 +47,7 @@ export const dataflowStarCommand: ReplCommand = {
 		output.stdout(mermaid);
 		try {
 			clipboard.writeSync(mermaid);
-			output.stdout(formatInfo(output, 'mermaid url'));
+			output.stdout(formatInfo(output, 'mermaid url', result.dataflow['.meta'].timing));
 		} catch(e) { /* do nothing this is a service thing */ }
 	}
 };

src/cli/repl/server/connection.ts

@@ -154,7 +154,7 @@ export class FlowRServerConnection {
 	private async sendFileAnalysisResponse(results: Partial<PipelineOutput<typeof DEFAULT_SLICING_PIPELINE>>, message: FileAnalysisRequestMessage): Promise<void> {
 		let cfg: ControlFlowInformation | undefined = undefined;
 		if(message.cfg) {
-			cfg = extractCFG(results.normalize as NormalizedAst);
+			cfg = extractCFG(results.normalize as NormalizedAst, results.dataflow?.graph);
 		}
 
 		const config = (): QuadSerializationConfiguration => ({ context: message.filename ?? 'unknown', getId: defaultQuadIdGenerator() });

src/cli/script-core/statistics-helper-core.ts

@@ -69,7 +69,7 @@ export async function getStatsForSingleFile(options: StatsHelperCliOptions) {
 	if(stats.outputs.size === 1) {
 		if(options['dump-json']) {
 			const [, output] = [...stats.outputs.entries()][0];
-			const cfg = extractCFG(output.normalize);
+			const cfg = extractCFG(output.normalize, output.dataflow.graph);
 			statisticsFileProvider.append('output-json', 'parse', await printStepResult(PARSE_WITH_R_SHELL_STEP, output.parse, StepOutputFormat.Json));
 			statisticsFileProvider.append('output-json', 'normalize', await printStepResult(NORMALIZE, output.normalize, StepOutputFormat.Json));
 			statisticsFileProvider.append('output-json', 'dataflow', await printStepResult(STATIC_DATAFLOW, output.dataflow, StepOutputFormat.Json));

src/dataflow/extractor.ts

@@ -67,7 +67,7 @@ function resolveLinkToSideEffects(ast: NormalizedAst, graph: DataflowGraph) {
 		if(typeof s !== 'object') {
 			continue;
 		}
-		cfg ??= extractCFG(ast).graph;
+		cfg ??= extractCFG(ast, graph).graph;
 		/* this has to change whenever we add a new link to relations because we currently offer no abstraction for the type */
 		const potentials = identifyLinkToLastCallRelation(s.id, cfg, graph, s.linkTo);
 		for(const pot of potentials) {

src/dataflow/graph/dataflowgraph-builder.ts

@@ -198,8 +198,9 @@ export class DataflowGraphBuilder extends DataflowGraph {
 	/**
 	 * Adds a **read edge** (E1).
 	 *
-	 * @param from - Vertex/NodeId
-	 * @param to   - see from
+	 * @param from - NodeId of the source vertex
+	 * @param to   - Either a single or multiple target ids.
+	 *               If you pass multiple this will construct a single edge for each of them.
 	 */
 	public reads(from: NodeId, to: DataflowGraphEdgeTarget) {
 		return this.edgeHelper(from, to, EdgeType.Reads);
@@ -209,7 +210,7 @@ export class DataflowGraphBuilder extends DataflowGraph {
 	 * Adds a **defined-by edge** (E2), with from as defined variable, and to
 	 * as a variable/function contributing to its definition.
 	 *
-	 * @see reads for parameters.
+	 * @see {@link DataflowGraphBuilder#reads|reads} for parameters.
 	 */
 	public definedBy(from: NodeId, to: DataflowGraphEdgeTarget) {
 		return this.edgeHelper(from, to, EdgeType.DefinedBy);
@@ -218,7 +219,7 @@ export class DataflowGraphBuilder extends DataflowGraph {
 	/**
 	 * Adds a **call edge** (E5) with from as caller, and to as callee.
 	 *
-	 * @see reads for parameters.
+	 * @see {@link DataflowGraphBuilder#reads|reads} for parameters.
 	 */
 	public calls(from: NodeId, to: DataflowGraphEdgeTarget) {
 		return this.edgeHelper(from, to, EdgeType.Calls);
@@ -227,7 +228,7 @@ export class DataflowGraphBuilder extends DataflowGraph {
 	/**
 	 * Adds a **return edge** (E6) with from as function, and to as exit point.
 	 *
-	 * @see reads for parameters.
+	 * @see {@link DataflowGraphBuilder#reads|reads} for parameters.
 	 */
 	public returns(from: NodeId, to: DataflowGraphEdgeTarget) {
 		return this.edgeHelper(from, to, EdgeType.Returns);
@@ -236,7 +237,7 @@ export class DataflowGraphBuilder extends DataflowGraph {
 	/**
 	 * Adds a **defines-on-call edge** (E7) with from as variable, and to as its definition
 	 *
-	 * @see reads for parameters.
+	 * @see {@link DataflowGraphBuilder#reads|reads} for parameters.
 	 */
 	public definesOnCall(from: NodeId, to: DataflowGraphEdgeTarget) {
 		return this.edgeHelper(from, to, EdgeType.DefinesOnCall);
@@ -245,7 +246,7 @@ export class DataflowGraphBuilder extends DataflowGraph {
 	/**
 	 * Adds a **defined-by-on-call edge** with from as definition, and to as variable.
 	 *
-	 * @see reads for parameters.
+	 * @see {@link DataflowGraphBuilder#reads|reads} for parameters.
 	 */
 	public definedByOnCall(from: NodeId, to: DataflowGraphEdgeTarget) {
 		return this.edgeHelper(from, to, EdgeType.DefinedByOnCall);
@@ -254,7 +255,7 @@ export class DataflowGraphBuilder extends DataflowGraph {
 	/**
 	 * Adds an **argument edge** (E9) with from as function call, and to as argument.
 	 *
-	 * @see reads for parameters.
+	 * @see {@link DataflowGraphBuilder#reads|reads} for parameters.
 	 */
 	public argument(from: NodeId, to: DataflowGraphEdgeTarget) {
 		return this.edgeHelper(from, to, EdgeType.Argument);
@@ -263,7 +264,7 @@ export class DataflowGraphBuilder extends DataflowGraph {
 	/**
 	 * Adds a **non-standard evaluation edge** with from as vertex, and to as vertex.
 	 *
-	 * @see reads for parameters.
+	 * @see {@link DataflowGraphBuilder#reads|reads} for parameters.
 	 */
 	public nse(from: NodeId, to: DataflowGraphEdgeTarget) {
 		return this.edgeHelper(from, to, EdgeType.NonStandardEvaluation);
@@ -272,13 +273,12 @@ export class DataflowGraphBuilder extends DataflowGraph {
 	/**
 	 * Adds a **side-effect-on-call edge** with from as vertex, and to as vertex.
 	 *
-	 * @see reads for parameters.
+	 * @see {@link DataflowGraphBuilder#reads|reads} for parameters.
 	 */
 	public sideEffectOnCall(from: NodeId, to: DataflowGraphEdgeTarget) {
 		return this.edgeHelper(from, to, EdgeType.SideEffectOnCall);
 	}
 
-
 	/**
 	 * explicitly overwrite the root ids of the graph,
 	 * this is just an easier variant in case you working with a lot of functions this saves you a lot of `false` flags.

src/dataflow/graph/edge.ts

@@ -6,7 +6,6 @@ export interface DataflowGraphEdge {
 	types: EdgeTypeBits
 }
 
-
 /**
  * Represents the relationship between the source and the target vertex in the dataflow graph.
  * The actual value is represented as a bitmask so use {@link edgeTypesToNames} to get something more human-readable.
@@ -117,8 +116,8 @@ export const enum TraverseEdge {
  *
  * Counterpart of {@link edgeDoesNotIncludeType}.
  */
-export function edgeIncludesType(type: EdgeTypeBits, types: EdgeTypeBits): boolean {
-	return (types & type) !== 0;
+export function edgeIncludesType(type: EdgeTypeBits, typesToInclude: EdgeTypeBits): boolean {
+	return (typesToInclude & type) !== 0;
 }
 
 /**

src/dataflow/graph/vertex.ts

@@ -23,7 +23,7 @@ export const ValidVertexTypeReverse = Object.fromEntries(Object.entries(VertexTy
  */
 export interface ContainerLeafIndex {
 	/**
-	 * Destinctive lexeme of index e.g 'name' for `list(name = 'John')`
+	 * Distinctive lexeme of index e.g. 'name' for `list(name = 'John')`
 	 */
 	readonly lexeme: string,
 
@@ -222,35 +222,35 @@ export type DataflowGraphVertices<Vertex extends DataflowGraphVertexInfo = Dataf
 /**
  * Check if the given vertex is a {@link DataflowGraphVertexValue|value vertex}.
  */
-export function isValueVertex(vertex: DataflowGraphVertexBase): vertex is DataflowGraphVertexValue {
-	return vertex.tag === VertexType.Value;
+export function isValueVertex(vertex?: DataflowGraphVertexBase): vertex is DataflowGraphVertexValue {
+	return vertex?.tag === VertexType.Value;
 }
 
 /**
  * Check if the given vertex is a {@link DataflowGraphVertexUse|use vertex}.
  */
-export function isUseVertex(vertex: DataflowGraphVertexBase): vertex is DataflowGraphVertexUse {
-	return vertex.tag === VertexType.Use;
+export function isUseVertex(vertex?: DataflowGraphVertexBase): vertex is DataflowGraphVertexUse {
+	return vertex?.tag === VertexType.Use;
 }
 
 /**
  * Check if the given vertex is a {@link DataflowGraphVertexFunctionCall|function call vertex}.
  */
-export function isFunctionCallVertex(vertex: DataflowGraphVertexBase): vertex is DataflowGraphVertexFunctionCall {
-	return vertex.tag === VertexType.FunctionCall;
+export function isFunctionCallVertex(vertex?: DataflowGraphVertexBase): vertex is DataflowGraphVertexFunctionCall {
+	return vertex?.tag === VertexType.FunctionCall;
 }
 
 /**
  * Check if the given vertex is a {@link DataflowGraphVertexVariableDefinition|variable definition vertex}.
  */
-export function isVariableDefinitionVertex(vertex: DataflowGraphVertexBase): vertex is DataflowGraphVertexVariableDefinition {
-	return vertex.tag === VertexType.VariableDefinition;
+export function isVariableDefinitionVertex(vertex?: DataflowGraphVertexBase): vertex is DataflowGraphVertexVariableDefinition {
+	return vertex?.tag === VertexType.VariableDefinition;
 }
 
 /**
  * Check if the given vertex is a {@link DataflowGraphVertexFunctionDefinition|function definition vertex}.
  */
-export function isFunctionDefinitionVertex(vertex: DataflowGraphVertexBase): vertex is DataflowGraphVertexFunctionDefinition {
-	return vertex.tag === VertexType.FunctionDefinition;
+export function isFunctionDefinitionVertex(vertex?: DataflowGraphVertexBase): vertex is DataflowGraphVertexFunctionDefinition {
+	return vertex?.tag === VertexType.FunctionDefinition;
 }
 

src/dataflow/internal/process/functions/call/built-in/built-in-expression-list.ts

@@ -162,7 +162,6 @@ export function processExpressionList<OtherInfo>(
 
 		processNextExpression(processed, environment, listEnvironments, remainingRead, nextGraph);
 
-
 		environment = exitPoints.length > 0 ? overwriteEnvironment(environment, processed.environment) : processed.environment;
 
 		const calledEnvs = linkFunctionCalls(nextGraph, data.completeAst.idMap, processed.graph);

src/documentation/doc-util/doc-cfg.ts

@@ -1,7 +1,7 @@
 import type { ControlFlowInformation } from '../../util/cfg/cfg';
 import { extractCFG } from '../../util/cfg/cfg';
 import { PipelineExecutor } from '../../core/pipeline-executor';
-import { DEFAULT_NORMALIZE_PIPELINE } from '../../core/steps/pipeline/default-pipelines';
+import { DEFAULT_DATAFLOW_PIPELINE } from '../../core/steps/pipeline/default-pipelines';
 import { requestFromInput } from '../../r-bridge/retriever';
 import type { RShell } from '../../r-bridge/shell';
 import type { NormalizedAst } from '../../r-bridge/lang-4.x/ast/model/processing/decorate';
@@ -10,12 +10,12 @@ export async function getCfg(shell: RShell, code: string): Promise<{
 	info: ControlFlowInformation,
 	ast:  NormalizedAst
 }> {
-	const steps = await new PipelineExecutor(DEFAULT_NORMALIZE_PIPELINE, {
+	const steps = await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, {
 		parser:  shell,
 		request: requestFromInput(code)
 	}).allRemainingSteps();
 	return {
-		info: extractCFG(steps.normalize),
+		info: extractCFG(steps.normalize, steps.dataflow?.graph),
 		ast:  steps.normalize
 	};
 }

src/queries/catalog/call-context-query/call-context-query-executor.ts

@@ -195,7 +195,7 @@ export function executeCallContextQueries({ dataflow: { graph }, ast }: BasicQue
 
 	let cfg = undefined;
 	if(requiresCfg) {
-		cfg = extractCFG(ast);
+		cfg = extractCFG(ast, graph);
 	}
 
 	const queriesWhichWantAliases = promotedQueries.filter(q => q.includeAliases);

src/queries/catalog/happens-before-query/happens-before-query-executor.ts

@@ -9,10 +9,10 @@ import { extractCFG } from '../../../util/cfg/cfg';
 import { happensBefore } from '../../../util/cfg/happens-before';
 import { slicingCriterionToId } from '../../../slicing/criterion/parse';
 
-export function executeHappensBefore({ ast }: BasicQueryData, queries: readonly HappensBeforeQuery[]): HappensBeforeQueryResult {
+export function executeHappensBefore({ ast, dataflow }: BasicQueryData, queries: readonly HappensBeforeQuery[]): HappensBeforeQueryResult {
 	const start = Date.now();
 	const results: Record<string, Ternary> = {};
-	const cfg = extractCFG(ast);
+	const cfg = extractCFG(ast, dataflow.graph);
 	for(const query of queries) {
 		const { a, b } = query;
 		const fingerprint = `${a}<${b}`;