Project Builder API

src/cli/flowr.ts

@@ -11,24 +11,23 @@ import { NetServer, WebSocketServerWrapper } from './repl/server/net';
 import { flowrVersion } from '../util/version';
 import commandLineUsage from 'command-line-usage';
 import { log, LogLevel } from '../util/log';
-import { bold, ColorEffect, Colors, FontStyles, formatter, italic, setFormatter, voidFormatter } from '../util/text/ansi';
+import { FontStyles, formatter, italic, setFormatter, voidFormatter } from '../util/text/ansi';
 import commandLineArgs from 'command-line-args';
 import type { EngineConfig, FlowrConfigOptions, KnownEngines } from '../config';
-import { amendConfig, getConfig, getEngineConfig, parseConfig } from '../config';
+import { amendConfig, getConfig, parseConfig } from '../config';
 import { guard } from '../util/assert';
 import type { ScriptInformation } from './common/scripts-info';
 import { scripts } from './common/scripts-info';
-import { RShell, RShellReviveOptions } from '../r-bridge/shell';
 import { waitOnScript } from './repl/execute';
 import { standardReplOutput } from './repl/commands/repl-main';
 import { repl, replProcessAnswer } from './repl/core';
 import { printVersionInformation } from './repl/commands/repl-version';
 import { printVersionRepl } from './repl/print-version';
 import { defaultConfigFile, flowrMainOptionDefinitions, getScriptsText } from './flowr-main-options';
-import { TreeSitterExecutor } from '../r-bridge/lang-4.x/tree-sitter/tree-sitter-executor';
 import type { KnownParser } from '../r-bridge/parser';
 import fs from 'fs';
 import path from 'path';
+import { retrieveEngineInstances } from '../engines';
 
 export const toolName = 'flowr';
 
@@ -137,33 +136,6 @@ function createConfig(): FlowrConfigOptions {
 	return config;
 }
 
-
-async function retrieveEngineInstances(config: FlowrConfigOptions): Promise<{ engines: KnownEngines, default: keyof KnownEngines }> {
-	const engines: KnownEngines = {};
-	if(getEngineConfig(config, 'r-shell')) {
-		// we keep an active shell session to allow other parse investigations :)
-		engines['r-shell'] = new RShell(getEngineConfig(config, 'r-shell'), {
-			revive:   RShellReviveOptions.Always,
-			onRevive: (code, signal) => {
-				const signalText = signal == null ? '' : ` and signal ${signal}`;
-				console.log(formatter.format(`R process exited with code ${code}${signalText}. Restarting...`, { color: Colors.Magenta, effect: ColorEffect.Foreground }));
-				console.log(italic(`If you want to exit, press either Ctrl+C twice, or enter ${bold(':quit')}`));
-			}
-		});
-	}
-	if(getEngineConfig(config, 'tree-sitter')) {
-		await TreeSitterExecutor.initTreeSitter(getEngineConfig(config, 'tree-sitter'));
-		engines['tree-sitter'] = new TreeSitterExecutor();
-	}
-	let defaultEngine = config.defaultEngine;
-	if(!defaultEngine || !engines[defaultEngine]) {
-		// if a default engine isn't specified, we just take the first one we have
-		defaultEngine = Object.keys(engines)[0] as keyof KnownEngines;
-	}
-	log.info(`Using engines ${Object.keys(engines).join(', ')} with default ${defaultEngine}`);
-	return { engines, default: defaultEngine };
-}
-
 function hookSignalHandlers(engines: { engines: KnownEngines; default: keyof KnownEngines }) {
 	const end = () => {
 		if(options.execute === undefined) {

src/cli/repl/commands/repl-cfg.ts

@@ -1,35 +1,22 @@
-import type { ReplCommand, ReplOutput } from './repl-main';
-import { extractCfg } from '../../../control-flow/extract-cfg';
-import { createDataflowPipeline } from '../../../core/steps/pipeline/default-pipelines';
-import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever';
+import type { ReplCodeCommand, ReplOutput } from './repl-main';
+import { fileProtocol } from '../../../r-bridge/retriever';
 import { cfgToMermaid, cfgToMermaidUrl } from '../../../util/mermaid/cfg';
-import type { KnownParser } from '../../../r-bridge/parser';
 import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi';
 import type { ControlFlowInformation } from '../../../control-flow/control-flow-graph';
 import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate';
 import type { CfgSimplificationPassName } from '../../../control-flow/cfg-simplification';
 import { DefaultCfgSimplificationOrder } from '../../../control-flow/cfg-simplification';
-import type { FlowrConfigOptions } from '../../../config';
-
-async function controlflow(parser: KnownParser, remainingLine: string, config: FlowrConfigOptions) {
-	return await createDataflowPipeline(parser, {
-		request: requestFromInput(remainingLine.trim())
-	}, config).allRemainingSteps();
-}
-
-function handleString(code: string): string {
-	return code.startsWith('"') ? JSON.parse(code) as string : code;
-}
+import type { FlowrAnalysisProvider } from '../../../project/flowr-analyzer';
+import { handleString } from '../core';
 
 function formatInfo(out: ReplOutput, type: string): string {
 	return out.formatter.format(`Copied ${type} to clipboard.`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic });
 }
 
-async function produceAndPrintCfg(shell: KnownParser, remainingLine: string, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string, config: FlowrConfigOptions) {
-	const result = await controlflow(shell, handleString(remainingLine), config);
-
-	const cfg = extractCfg(result.normalize, config, result.dataflow.graph, [...DefaultCfgSimplificationOrder, ...simplifications]);
-	const mermaid = cfgConverter(cfg, result.normalize);
+async function produceAndPrintCfg(analyzer: FlowrAnalysisProvider, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string) {
+	const cfg = await analyzer.controlflow([...DefaultCfgSimplificationOrder, ...simplifications]);
+	const normalizedAst = await analyzer.normalize();
+	const mermaid = cfgConverter(cfg, normalizedAst);
 	output.stdout(mermaid);
 	try {
 		const clipboard = await import('clipboardy');
@@ -39,45 +26,53 @@ async function produceAndPrintCfg(shell: KnownParser, remainingLine: string, out
 	}
 }
 
-export const controlflowCommand: ReplCommand = {
+export const controlflowCommand: ReplCodeCommand = {
 	description:  `Get mermaid code for the control-flow graph of R code, start with '${fileProtocol}' to indicate a file`,
+	usesAnalyzer: true,
 	usageExample: ':controlflow',
 	aliases:      [ 'cfg', 'cf' ],
 	script:       false,
-	fn:           async({ output, parser, remainingLine, config }) => {
-		await produceAndPrintCfg(parser, remainingLine, output, [], cfgToMermaid, config);
+	argsParser:   (args: string) => handleString(args),
+	fn:           async({ output, analyzer }) => {
+		await produceAndPrintCfg(analyzer, output, [], cfgToMermaid);
 	}
 };
 
 
-export const controlflowStarCommand: ReplCommand = {
+export const controlflowStarCommand: ReplCodeCommand = {
 	description:  'Returns the URL to mermaid.live',
+	usesAnalyzer: true,
 	usageExample: ':controlflow*',
 	aliases:      [ 'cfg*', 'cf*' ],
 	script:       false,
-	fn:           async({ output, parser, remainingLine, config }) => {
-		await produceAndPrintCfg(parser, remainingLine, output, [], cfgToMermaidUrl, config);
+	argsParser:   (args: string) => handleString(args),
+	fn:           async({ output, analyzer }) => {
+		await produceAndPrintCfg(analyzer, output, [], cfgToMermaidUrl);
 	}
 };
 
 
-export const controlflowBbCommand: ReplCommand = {
+export const controlflowBbCommand: ReplCodeCommand = {
 	description:  `Get mermaid code for the control-flow graph with basic blocks, start with '${fileProtocol}' to indicate a file`,
+	usesAnalyzer: true,
 	usageExample: ':controlflowbb',
 	aliases:      [ 'cfgb', 'cfb' ],
 	script:       false,
-	fn:           async({ output, parser, remainingLine, config }) => {
-		await produceAndPrintCfg(parser, remainingLine, output, ['to-basic-blocks'], cfgToMermaid, config);
+	argsParser:   (args: string) => handleString(args),
+	fn:           async({ output, analyzer }) => {
+		await produceAndPrintCfg(analyzer, output, ['to-basic-blocks'], cfgToMermaid);
 	}
 };
 
 
-export const controlflowBbStarCommand: ReplCommand = {
+export const controlflowBbStarCommand: ReplCodeCommand = {
 	description:  'Returns the URL to mermaid.live',
+	usesAnalyzer: true,
 	usageExample: ':controlflowbb*',
 	aliases:      [ 'cfgb*', 'cfb*' ],
 	script:       false,
-	fn:           async({ output, parser, remainingLine, config }) => {
-		await produceAndPrintCfg(parser, remainingLine, output, ['to-basic-blocks' ], cfgToMermaidUrl, config);
+	argsParser:   (args: string) => handleString(args),
+	fn:           async({ output, analyzer }) => {
+		await produceAndPrintCfg(analyzer, output, ['to-basic-blocks' ], cfgToMermaidUrl);
 	}
 };

src/cli/repl/commands/repl-commands.ts

@@ -1,6 +1,6 @@
 import { quitCommand } from './repl-quit';
 import { stdioCaptureProcessor, waitOnScript } from '../execute';
-import type { ReplCommand } from './repl-main';
+import type { ReplBaseCommand, ReplCodeCommand, ReplCommand } from './repl-main';
 import { rawPrompt } from '../prompt';
 import { versionCommand } from './repl-version';
 import { parseCommand } from './repl-parse';
@@ -21,7 +21,7 @@ import { scripts } from '../../common/scripts-info';
 import { lineageCommand } from './repl-lineage';
 import { queryCommand, queryStarCommand } from './repl-query';
 
-function printHelpForScript(script: [string, ReplCommand], f: OutputFormatter, starredVersion?: ReplCommand): string {
+function printHelpForScript(script: [string, ReplBaseCommand], f: OutputFormatter, starredVersion?: ReplBaseCommand): string {
 	let base = `  ${bold(padCmd(':' + script[0] + (starredVersion ? '[*]' : '')), f)}${script[1].description}`;
 	if(starredVersion) {
 		base += ` (star: ${starredVersion.description})`;
@@ -49,6 +49,7 @@ function printCommandHelp(formatter: OutputFormatter) {
 
 export const helpCommand: ReplCommand = {
 	description:  'Show help information',
+	usesAnalyzer: false,
 	script:       false,
 	usageExample: ':help',
 	aliases:      [ 'h', '?' ],
@@ -79,7 +80,7 @@ You can combine commands by separating them with a semicolon ${bold(';',output.f
 /**
  * All commands that should be available in the REPL.
  */
-const _commands: Record<string, ReplCommand> = {
+const _commands: Record<string, ReplCommand | ReplCodeCommand> = {
 	'help':            helpCommand,
 	'quit':            quitCommand,
 	'version':         versionCommand,
@@ -122,6 +123,7 @@ export function getReplCommands() {
 				aliases:      [],
 				script:       true,
 				usageExample: `:${script} --help`,
+				usesAnalyzer: false,
 				fn:           async({ output, remainingLine }) => {
 					// check if the target *module* exists in the current directory, else try two dirs up, otherwise, fail with a message
 					let path = `${__dirname}/${target}`;
@@ -177,7 +179,7 @@ function initCommandMapping() {
  * Get the command for a given command name or alias.
  * @param command - The name of the command (without the leading `:`)
  */
-export function getCommand(command: string): ReplCommand | undefined {
+export function getCommand(command: string): ReplCodeCommand | ReplCommand | undefined {
 	if(commandMapping === undefined) {
 		initCommandMapping();
 	}

src/cli/repl/commands/repl-dataflow.ts

@@ -1,93 +1,88 @@
-import type { ReplCommand, ReplOutput } from './repl-main';
-import { createDataflowPipeline } from '../../../core/steps/pipeline/default-pipelines';
-import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever';
+import type { ReplCodeCommand, ReplOutput } from './repl-main';
+import { fileProtocol } from '../../../r-bridge/retriever';
 import { graphToMermaid, graphToMermaidUrl } from '../../../util/mermaid/dfg';
-import type { KnownParser } from '../../../r-bridge/parser';
 import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi';
-import type { FlowrConfigOptions } from '../../../config';
+import type { PipelinePerStepMetaInformation } from '../../../core/steps/pipeline/pipeline';
+import { handleString } from '../core';
 
-/**
- * Obtain the dataflow graph using a known parser (such as the {@link RShell} or {@link TreeSitterExecutor}).
- */
-async function replGetDataflow(config: FlowrConfigOptions, parser: KnownParser, code: string) {
-	return await createDataflowPipeline(parser, {
-		request: requestFromInput(code.trim())
-	}, config).allRemainingSteps();
+function formatInfo(out: ReplOutput, type: string, meta: PipelinePerStepMetaInformation ): string {
+	return out.formatter.format(`Copied ${type} to clipboard (dataflow: ${meta['.meta'].timing + 'ms'}).`,
+		{ color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic });
 }
 
-function handleString(code: string): string {
-	return code.startsWith('"') ? JSON.parse(code) as string : code;
-}
-
-function formatInfo(out: ReplOutput, type: string, timing: number): string {
-	return out.formatter.format(`Copied ${type} to clipboard (dataflow: ${timing}ms).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic });
-}
-
-export const dataflowCommand: ReplCommand = {
+export const dataflowCommand: ReplCodeCommand = {
 	description:  `Get mermaid code for the dataflow graph, start with '${fileProtocol}' to indicate a file`,
+	usesAnalyzer: true,
 	usageExample: ':dataflow',
 	aliases:      [ 'd', 'df' ],
 	script:       false,
-	fn:           async({ output, parser, remainingLine, config }) => {
-		const result = await replGetDataflow(config, parser, handleString(remainingLine));
-		const mermaid = graphToMermaid({ graph: result.dataflow.graph, includeEnvironments: false }).string;
+	argsParser:   (args: string) => handleString(args),
+	fn:           async({ output, analyzer }) => {
+		const result = await analyzer.dataflow();
+		const mermaid = graphToMermaid({ graph: result.graph, includeEnvironments: false }).string;
 		output.stdout(mermaid);
 		try {
 			const clipboard = await import('clipboardy');
 			clipboard.default.writeSync(mermaid);
-			output.stdout(formatInfo(output, 'mermaid code', result.dataflow['.meta'].timing));
+			output.stdout(formatInfo(output, 'mermaid code', result));
 		} catch{ /* do nothing this is a service thing */ }
 	}
 };
 
-export const dataflowStarCommand: ReplCommand = {
+export const dataflowStarCommand: ReplCodeCommand = {
 	description:  'Returns the URL to mermaid.live',
+	usesAnalyzer: true,
 	usageExample: ':dataflow*',
 	aliases:      [ 'd*', 'df*' ],
 	script:       false,
-	fn:           async({ output, parser, remainingLine, config }) => {
-		const result = await replGetDataflow(config, parser, handleString(remainingLine));
-		const mermaid = graphToMermaidUrl(result.dataflow.graph, false);
+	argsParser:   (args: string) => handleString(args),
+	fn:           async({ output, analyzer }) => {
+		const result = await analyzer.dataflow();
+		const mermaid = graphToMermaidUrl(result.graph, false);
 		output.stdout(mermaid);
 		try {
 			const clipboard = await import('clipboardy');
 			clipboard.default.writeSync(mermaid);
-			output.stdout(formatInfo(output, 'mermaid url', result.dataflow['.meta'].timing));
+			output.stdout(formatInfo(output, 'mermaid url', result));
 		} catch{ /* do nothing this is a service thing */ }
 	}
 };
 
 
-export const dataflowSimplifiedCommand: ReplCommand = {
+export const dataflowSimplifiedCommand: ReplCodeCommand = {
 	description:  `Get mermaid code for the simplified dataflow graph, start with '${fileProtocol}' to indicate a file`,
+	usesAnalyzer: true,
 	usageExample: ':dataflowsimple',
 	aliases:      [ 'ds', 'dfs' ],
 	script:       false,
-	fn:           async({ output, parser, remainingLine, config }) => {
-		const result = await replGetDataflow(config, parser, handleString(remainingLine));
-		const mermaid = graphToMermaid({ graph: result.dataflow.graph, includeEnvironments: false, simplified: true }).string;
+	argsParser:   (args: string) => handleString(args),
+	fn:           async({ output, analyzer }) => {
+		const result = await analyzer.dataflow();
+		const mermaid = graphToMermaid({ graph: result.graph, includeEnvironments: false, simplified: true }).string;
 		output.stdout(mermaid);
 		try {
 			const clipboard = await import('clipboardy');
 			clipboard.default.writeSync(mermaid);
-			output.stdout(formatInfo(output, 'mermaid code', result.dataflow['.meta'].timing));
+			output.stdout(formatInfo(output, 'mermaid code', result));
 		} catch{ /* do nothing this is a service thing */ }
 	}
 };
 
-export const dataflowSimpleStarCommand: ReplCommand = {
+export const dataflowSimpleStarCommand: ReplCodeCommand = {
 	description:  'Returns the URL to mermaid.live',
+	usesAnalyzer: true,
 	usageExample: ':dataflowsimple*',
 	aliases:      [ 'ds*', 'dfs*' ],
 	script:       false,
-	fn:           async({ output, parser, remainingLine, config }) => {
-		const result = await replGetDataflow(config, parser, handleString(remainingLine));
-		const mermaid = graphToMermaidUrl(result.dataflow.graph, false, undefined, true);
+	argsParser:   (args: string) => handleString(args),
+	fn:           async({ output, analyzer }) => {
+		const result = await analyzer.dataflow();
+		const mermaid = graphToMermaidUrl(result.graph, false, undefined, true);
 		output.stdout(mermaid);
 		try {
 			const clipboard = await import('clipboardy');
 			clipboard.default.writeSync(mermaid);
-			output.stdout(formatInfo(output, 'mermaid url', result.dataflow['.meta'].timing));
+			output.stdout(formatInfo(output, 'mermaid url', result));
 		} catch{ /* do nothing this is a service thing */ }
 	}
 };

src/cli/repl/commands/repl-execute.ts

@@ -27,6 +27,7 @@ async function executeRShellCommand(output: ReplOutput, shell: RShell, statement
 
 export const executeCommand: ReplCommand = {
 	description:  'Execute the given code as R code (essentially similar to using now command). This requires the `--r-session-access` flag to be set and requires the r-shell engine.',
+	usesAnalyzer: false,
 	usageExample: ':execute',
 	aliases:      [ 'e', 'r' ],
 	script:       false,

src/cli/repl/commands/repl-lineage.ts

@@ -67,6 +67,7 @@ export function getLineage(criterion: SingleSlicingCriterion, graph: DataflowGra
 
 export const lineageCommand: ReplCommand = {
 	description:  'Get the lineage of an R object',
+	usesAnalyzer: false,
 	usageExample: ':lineage',
 	aliases:      ['lin'],
 	script:       false,