From dc69dd7c7d9384c39f34eb8a09a9f037add40d8e Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Tue, 24 Jun 2025 10:39:21 +0200 Subject: [PATCH 01/70] feat(builder): add first experiment --- src/project/flowr-analyzer-builder.ts | 26 +++++++++++++++++ src/project/flowr-analyzer.ts | 40 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 src/project/flowr-analyzer-builder.ts create mode 100644 src/project/flowr-analyzer.ts diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts new file mode 100644 index 00000000000..f87e1b729bf --- /dev/null +++ b/src/project/flowr-analyzer-builder.ts @@ -0,0 +1,26 @@ +import type { FlowrConfigOptions } from '../config'; +import { amendConfig, cloneConfig, defaultConfigOptions } from '../config'; +import type { DeepWritable } from 'ts-essentials'; +import type { RParseRequests } from '../r-bridge/retriever'; +import { FlowrAnalyzer } from './flowr-analyzer'; + +export class FlowrAnalyzerBuilder { + private flowrConfig: FlowrConfigOptions = cloneConfig(defaultConfigOptions); + private request: RParseRequests; + + public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions) : FlowrAnalyzerBuilder { + amendConfig(this.flowrConfig, func); + return this; + } + + constructor(request: RParseRequests) { + this.request = request; + } + + public build(): FlowrAnalyzer { + return new FlowrAnalyzer( + this.flowrConfig, + this.request + ); + } +} \ No newline at end of file diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts new file mode 100644 index 00000000000..43c8e9eee9e --- /dev/null +++ b/src/project/flowr-analyzer.ts @@ -0,0 +1,40 @@ +import type { FlowrConfigOptions } from '../config'; +import { cloneConfig, defaultConfigOptions } from '../config'; +import type { RParseRequests } from '../r-bridge/retriever'; +import { requestFromInput } from '../r-bridge/retriever'; +import { PipelineExecutor } from '../core/pipeline-executor'; +import { DEFAULT_DATAFLOW_PIPELINE } from '../core/steps/pipeline/default-pipelines'; +import { RShell } from '../r-bridge/shell'; +import type { PipelineOutput } from '../core/steps/pipeline/pipeline'; +import { FlowrAnalyzerBuilder } from './flowr-analyzer-builder'; +import { graphToMermaidUrl } from '../util/mermaid/dfg'; + +export class FlowrAnalyzer { + private flowrConfig: FlowrConfigOptions = cloneConfig(defaultConfigOptions); + private request: RParseRequests; + + constructor(config: FlowrConfigOptions, request: RParseRequests) { + this.flowrConfig = config; + this.request = request; + } + + public dataflow() : Promise> { + return new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, { + parser: new RShell(), + request: this.request + }, this.flowrConfig).allRemainingSteps(); + } +} + +async function main() { + const result = await new FlowrAnalyzerBuilder(requestFromInput('x <- 1')) + .amendConfig(c => { + c.solver.pointerTracking = true; + return c; + }) + .build() + .dataflow(); + console.error(graphToMermaidUrl(result.dataflow.graph)); +} + +void main(); \ No newline at end of file From fbafaab3a8f1b7882930cd74d595f8d810dcae95 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Tue, 24 Jun 2025 14:24:58 +0200 Subject: [PATCH 02/70] feat(builder): add engine and query support --- src/cli/flowr.ts | 34 ++----------- src/engines.ts | 32 +++++++++++++ src/project/flowr-analyzer-builder.ts | 19 ++++++-- src/project/flowr-analyzer.ts | 69 +++++++++++++++++++++------ 4 files changed, 104 insertions(+), 50 deletions(-) create mode 100644 src/engines.ts diff --git a/src/cli/flowr.ts b/src/cli/flowr.ts index 5c2bed16cba..19ead3036e4 100644 --- a/src/cli/flowr.ts +++ b/src/cli/flowr.ts @@ -11,24 +11,23 @@ import { NetServer, WebSocketServerWrapper } from './repl/server/net'; import { flowrVersion } from '../util/version'; import commandLineUsage from 'command-line-usage'; import { log, LogLevel } from '../util/log'; -import { bold, ColorEffect, Colors, FontStyles, formatter, italic, setFormatter, voidFormatter } from '../util/text/ansi'; +import { FontStyles, formatter, italic, setFormatter, voidFormatter } from '../util/text/ansi'; import commandLineArgs from 'command-line-args'; import type { EngineConfig, FlowrConfigOptions, KnownEngines } from '../config'; -import { amendConfig, getConfig, getEngineConfig, parseConfig } from '../config'; +import { amendConfig, getConfig, parseConfig } from '../config'; import { guard } from '../util/assert'; import type { ScriptInformation } from './common/scripts-info'; import { scripts } from './common/scripts-info'; -import { RShell, RShellReviveOptions } from '../r-bridge/shell'; import { waitOnScript } from './repl/execute'; import { standardReplOutput } from './repl/commands/repl-main'; import { repl, replProcessAnswer } from './repl/core'; import { printVersionInformation } from './repl/commands/repl-version'; import { printVersionRepl } from './repl/print-version'; import { defaultConfigFile, flowrMainOptionDefinitions, getScriptsText } from './flowr-main-options'; -import { TreeSitterExecutor } from '../r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; import type { KnownParser } from '../r-bridge/parser'; import fs from 'fs'; import path from 'path'; +import { retrieveEngineInstances } from '../engines'; export const toolName = 'flowr'; @@ -137,33 +136,6 @@ function createConfig() : FlowrConfigOptions { return config; } - -async function retrieveEngineInstances(config: FlowrConfigOptions): Promise<{ engines: KnownEngines, default: keyof KnownEngines }> { - const engines: KnownEngines = {}; - if(getEngineConfig(config, 'r-shell')) { - // we keep an active shell session to allow other parse investigations :) - engines['r-shell'] = new RShell(getEngineConfig(config, 'r-shell'), { - revive: RShellReviveOptions.Always, - onRevive: (code, signal) => { - const signalText = signal == null ? '' : ` and signal ${signal}`; - console.log(formatter.format(`R process exited with code ${code}${signalText}. Restarting...`, { color: Colors.Magenta, effect: ColorEffect.Foreground })); - console.log(italic(`If you want to exit, press either Ctrl+C twice, or enter ${bold(':quit')}`)); - } - }); - } - if(getEngineConfig(config, 'tree-sitter')) { - await TreeSitterExecutor.initTreeSitter(getEngineConfig(config, 'tree-sitter')); - engines['tree-sitter'] = new TreeSitterExecutor(); - } - let defaultEngine = config.defaultEngine; - if(!defaultEngine || !engines[defaultEngine]) { - // if a default engine isn't specified, we just take the first one we have - defaultEngine = Object.keys(engines)[0] as keyof KnownEngines; - } - log.info(`Using engines ${Object.keys(engines).join(', ')} with default ${defaultEngine}`); - return { engines, default: defaultEngine }; -} - function hookSignalHandlers(engines: { engines: KnownEngines; default: keyof KnownEngines }) { const end = () => { if(options.execute === undefined) { diff --git a/src/engines.ts b/src/engines.ts new file mode 100644 index 00000000000..7d87f734ca8 --- /dev/null +++ b/src/engines.ts @@ -0,0 +1,32 @@ +import type { FlowrConfigOptions, KnownEngines } from './config'; +import { getEngineConfig } from './config'; +import { RShell, RShellReviveOptions } from './r-bridge/shell'; +import { bold, ColorEffect, Colors, formatter, italic } from './util/text/ansi'; +import { TreeSitterExecutor } from './r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; +import { log } from './util/log'; + +export async function retrieveEngineInstances(config: FlowrConfigOptions): Promise<{ engines: KnownEngines, default: keyof KnownEngines }> { + const engines: KnownEngines = {}; + if(getEngineConfig(config, 'r-shell')) { + // we keep an active shell session to allow other parse investigations :) + engines['r-shell'] = new RShell(getEngineConfig(config, 'r-shell'), { + revive: RShellReviveOptions.Always, + onRevive: (code, signal) => { + const signalText = signal == null ? '' : ` and signal ${signal}`; + console.log(formatter.format(`R process exited with code ${code}${signalText}. Restarting...`, { color: Colors.Magenta, effect: ColorEffect.Foreground })); + console.log(italic(`If you want to exit, press either Ctrl+C twice, or enter ${bold(':quit')}`)); + } + }); + } + if(getEngineConfig(config, 'tree-sitter')) { + await TreeSitterExecutor.initTreeSitter(getEngineConfig(config, 'tree-sitter')); + engines['tree-sitter'] = new TreeSitterExecutor(); + } + let defaultEngine = config.defaultEngine; + if(!defaultEngine || !engines[defaultEngine]) { + // if a default engine isn't specified, we just take the first one we have + defaultEngine = Object.keys(engines)[0] as keyof KnownEngines; + } + log.info(`Using engines ${Object.keys(engines).join(', ')} with default ${defaultEngine}`); + return { engines, default: defaultEngine }; +} diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index f87e1b729bf..3fba465495a 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -3,13 +3,20 @@ import { amendConfig, cloneConfig, defaultConfigOptions } from '../config'; import type { DeepWritable } from 'ts-essentials'; import type { RParseRequests } from '../r-bridge/retriever'; import { FlowrAnalyzer } from './flowr-analyzer'; +import { retrieveEngineInstances } from '../engines'; +import type { KnownParser } from '../r-bridge/parser'; export class FlowrAnalyzerBuilder { - private flowrConfig: FlowrConfigOptions = cloneConfig(defaultConfigOptions); - private request: RParseRequests; + private flowrConfig: DeepWritable = cloneConfig(defaultConfigOptions); + private readonly request: RParseRequests; public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions) : FlowrAnalyzerBuilder { - amendConfig(this.flowrConfig, func); + this.flowrConfig = amendConfig(this.flowrConfig, func); + return this; + } + + public setEngine(engine : 'tree-sitter' | 'r-shell') { + this.flowrConfig.defaultEngine = engine; return this; } @@ -17,9 +24,13 @@ export class FlowrAnalyzerBuilder { this.request = request; } - public build(): FlowrAnalyzer { + public async build(): Promise { + const engines = await retrieveEngineInstances(this.flowrConfig); + const parser = engines.engines[engines.default] as KnownParser; + return new FlowrAnalyzer( this.flowrConfig, + parser, this.request ); } diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 43c8e9eee9e..08446c37d41 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -2,39 +2,78 @@ import type { FlowrConfigOptions } from '../config'; import { cloneConfig, defaultConfigOptions } from '../config'; import type { RParseRequests } from '../r-bridge/retriever'; import { requestFromInput } from '../r-bridge/retriever'; -import { PipelineExecutor } from '../core/pipeline-executor'; -import { DEFAULT_DATAFLOW_PIPELINE } from '../core/steps/pipeline/default-pipelines'; -import { RShell } from '../r-bridge/shell'; +import type { DEFAULT_DATAFLOW_PIPELINE } from '../core/steps/pipeline/default-pipelines'; +import { createDataflowPipeline, createSlicePipeline } from '../core/steps/pipeline/default-pipelines'; import type { PipelineOutput } from '../core/steps/pipeline/pipeline'; import { FlowrAnalyzerBuilder } from './flowr-analyzer-builder'; import { graphToMermaidUrl } from '../util/mermaid/dfg'; +import type { KnownParser } from '../r-bridge/parser'; +import type { SlicingCriteria } from '../slicing/criterion/parse'; +import type { Queries, SupportedQueryTypes } from '../queries/query'; +import { executeQueries } from '../queries/query'; +import type { DataflowInformation } from '../dataflow/info'; +import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; export class FlowrAnalyzer { - private flowrConfig: FlowrConfigOptions = cloneConfig(defaultConfigOptions); - private request: RParseRequests; + private readonly flowrConfig: FlowrConfigOptions = cloneConfig(defaultConfigOptions); + private readonly request: RParseRequests; + private readonly parser: KnownParser; + private ast = undefined as unknown as NormalizedAst; + private dataflowInfo = undefined as unknown as DataflowInformation; - constructor(config: FlowrConfigOptions, request: RParseRequests) { + constructor(config: FlowrConfigOptions, parser: KnownParser, request: RParseRequests) { this.flowrConfig = config; this.request = request; + this.parser = parser; } - public dataflow() : Promise> { - return new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, { - parser: new RShell(), - request: this.request - }, this.flowrConfig).allRemainingSteps(); + public async dataflow() : Promise> { + const result = await createDataflowPipeline( + this.parser, + { request: this.request }, + this.flowrConfig).allRemainingSteps(); + this.dataflowInfo = result.dataflow; + this.ast = result.normalize; + return result; + } + + public async slice(criterion: SlicingCriteria) { + return createSlicePipeline( + this.parser, + { + request: this.request, + criterion: criterion + }, + this.flowrConfig).allRemainingSteps(); + } + + public async query(query: Queries) { + if(!this.dataflowInfo) { + await this.dataflow(); + } + return executeQueries({ ast: this.ast, dataflow: this.dataflowInfo, config: this.flowrConfig }, query); } } async function main() { - const result = await new FlowrAnalyzerBuilder(requestFromInput('x <- 1')) + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('x <- 1\nfoo <- function(){}\nfoo()')) + .setEngine('r-shell') .amendConfig(c => { c.solver.pointerTracking = true; return c; }) - .build() - .dataflow(); - console.error(graphToMermaidUrl(result.dataflow.graph)); + .build(); + const result = await analyzer.dataflow(); + const query = await analyzer.query([{ + type: 'call-context', + kind: 'test-kind', + subkind: 'test-subkind', + callName: /foo/ + }]); + const slice = await analyzer.slice(['3@foo']); + console.log(graphToMermaidUrl(result.dataflow.graph)); + console.log(slice.reconstruct); + console.log(query); } void main(); \ No newline at end of file From 9a6abdc7e8df70bc2c61605bcdc26effa15b7ada Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Wed, 25 Jun 2025 00:04:52 +0200 Subject: [PATCH 03/70] feat(builder): fixes from review --- src/project/flowr-analyzer-builder.ts | 2 +- src/project/flowr-analyzer.ts | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 3fba465495a..2ea29e83f57 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -10,7 +10,7 @@ export class FlowrAnalyzerBuilder { private flowrConfig: DeepWritable = cloneConfig(defaultConfigOptions); private readonly request: RParseRequests; - public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions) : FlowrAnalyzerBuilder { + public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions) : this { this.flowrConfig = amendConfig(this.flowrConfig, func); return this; } diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 08446c37d41..b8da8ebcd42 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -1,10 +1,7 @@ import type { FlowrConfigOptions } from '../config'; -import { cloneConfig, defaultConfigOptions } from '../config'; import type { RParseRequests } from '../r-bridge/retriever'; import { requestFromInput } from '../r-bridge/retriever'; -import type { DEFAULT_DATAFLOW_PIPELINE } from '../core/steps/pipeline/default-pipelines'; import { createDataflowPipeline, createSlicePipeline } from '../core/steps/pipeline/default-pipelines'; -import type { PipelineOutput } from '../core/steps/pipeline/pipeline'; import { FlowrAnalyzerBuilder } from './flowr-analyzer-builder'; import { graphToMermaidUrl } from '../util/mermaid/dfg'; import type { KnownParser } from '../r-bridge/parser'; @@ -15,7 +12,7 @@ import type { DataflowInformation } from '../dataflow/info'; import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; export class FlowrAnalyzer { - private readonly flowrConfig: FlowrConfigOptions = cloneConfig(defaultConfigOptions); + private readonly flowrConfig: FlowrConfigOptions; private readonly request: RParseRequests; private readonly parser: KnownParser; private ast = undefined as unknown as NormalizedAst; @@ -27,7 +24,7 @@ export class FlowrAnalyzer { this.parser = parser; } - public async dataflow() : Promise> { + public async dataflow() { const result = await createDataflowPipeline( this.parser, { request: this.request }, @@ -76,4 +73,6 @@ async function main() { console.log(query); } -void main(); \ No newline at end of file +if(require.main === module) { + void main(); +} \ No newline at end of file From 7949d5de6b5c2b8fd48d29a9f1c428098aed7731 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Wed, 2 Jul 2025 20:34:53 +0200 Subject: [PATCH 04/70] feat(plugin): add FlowrAnalyzerPlugin and DESCRIPTION-file plugin --- .../flowr-analyzer-library-versions-plugin.ts | 152 ++++++++++++++++++ .../flowr-analyzer-loading-order-plugin.ts | 7 + .../plugins/flowr-analyzer-plugin-util.ts | 16 ++ src/project/plugins/flowr-analyzer-plugin.ts | 21 +++ .../plugins/flowr-analyzer-scoping-plugin.ts | 5 + 5 files changed, 201 insertions(+) create mode 100644 src/project/plugins/flowr-analyzer-library-versions-plugin.ts create mode 100644 src/project/plugins/flowr-analyzer-loading-order-plugin.ts create mode 100644 src/project/plugins/flowr-analyzer-plugin-util.ts create mode 100644 src/project/plugins/flowr-analyzer-plugin.ts create mode 100644 src/project/plugins/flowr-analyzer-scoping-plugin.ts diff --git a/src/project/plugins/flowr-analyzer-library-versions-plugin.ts b/src/project/plugins/flowr-analyzer-library-versions-plugin.ts new file mode 100644 index 00000000000..fe392d23a36 --- /dev/null +++ b/src/project/plugins/flowr-analyzer-library-versions-plugin.ts @@ -0,0 +1,152 @@ +import type { FlowrAnalyzerPlugin } from './flowr-analyzer-plugin'; +import { Range , SemVer } from 'semver'; +import type { FlowrAnalyzer } from '../flowr-analyzer'; +import type { FlowrConfigOptions } from '../../config'; +import type { PathLike } from 'fs'; +import fs from 'fs'; +import readline from 'readline'; +import { findFileFromRoot } from './flowr-analyzer-plugin-util'; + +export interface FlowrAnalyzerLibraryVersionsPlugin extends FlowrAnalyzerPlugin { + type: 'library-versions'; + libraries: Library[]; +} + +interface Library { + name: string; + version?: Range; + dependencies?: Library[]; + type: 'package' | 'system' | 'r'; + addInfo(name?: string, versionConstraint?: Range, type?: 'package' | 'system' | 'r', dependency?: []): void; + getInfo(): Library; +} + +function defaultLibraryAddInfo(this: Library, name?: string, versionConstraint?: Range, type?: 'package' | 'system' | 'r', dependency?: []): void { + if(name){ + this.name = name; + } + if(versionConstraint){ + this.version = versionConstraint; + } + if(type){ + this.type = type; + } + if(dependency){ + if(!this.dependencies){ + this.dependencies = dependency; + } else { + if(Array.isArray(dependency)){ + this.dependencies.push(...dependency); + } else { + this.dependencies.push(dependency); + } + } + } +} + +function defaultLibraryGetInfo(this: Library): Library { + return this; +} + +function parseVersionRange(constraint?: string, version?: string): Range | undefined { + if(constraint && version) { + return new Range(constraint + version); + } + if(version) { + return new Range(version); + } + return undefined; +} + +export async function findFilesForPlugin(files: PathLike | PathLike[] | undefined, findFilesMethod: (rootPath: PathLike) => Promise, rootPath: PathLike) : Promise { + if(files === undefined){ + const foundFiles: PathLike | PathLike[] | undefined = await findFilesMethod(rootPath); + if(foundFiles === undefined) { + console.log('Could not locate DESCRIPTION file!'); + return undefined; + } + return foundFiles; + } + return files; +} + +export const libraryVersionsDescriptionFilePlugin: FlowrAnalyzerLibraryVersionsPlugin & { + descriptionFile: PathLike | undefined; + rootPath: PathLike | undefined; + findDescriptionFile: (rootPath: PathLike) => Promise; + setDescriptionFile: (descriptionFile: PathLike) => void; + setRootPath: (rootpath: PathLike) => void; +} = { + name: 'library-versions-description-file', + description: 'This plugin does...', + version: new SemVer('0.1.0'), + type: 'library-versions', + libraries: [], + descriptionFile: undefined, + rootPath: undefined, + setRootPath(rootPath: PathLike): void { + this.rootPath = rootPath; + }, + async findDescriptionFile(rootPath: PathLike): Promise{ + return findFileFromRoot(rootPath, 'DESCRIPTION'); + }, + setDescriptionFile(descriptionFile: PathLike){ + this.descriptionFile = descriptionFile; + }, + async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { + console.log('Running Description-file Library Versions Plugin...'); + console.log(analyzer); + console.log(pluginConfig); + + this.descriptionFile = await findFilesForPlugin(this.descriptionFile, this.findDescriptionFile, this.rootPath ? this.rootPath : '') as PathLike | undefined; + + if(this.descriptionFile === undefined){ + return; + } + + try { + const readStream = fs.createReadStream(this.descriptionFile, { encoding: 'utf8' }); + const rl = readline.createInterface({ + input: readStream, + crlfDelay: Infinity + }); + console.log('Reading DESCRIPTION file...'); + + let inImports: boolean = false; + const importsRegex = new RegExp('^Imports:\\s*$'); + const versionRegex = new RegExp('^\\s*([a-zA-Z0-9._-]+)\\s*(\\((>=|<=|>|<|=)\\s*([0-9A-Za-z\\-+.*]+)\\))?,?\\s*$'); + const indentRegex = new RegExp('^\\s+'); + + for await (const line of rl) { + if(importsRegex.test(line)){ + inImports = true; + continue; + } + if(inImports){ + if(!indentRegex.test(line)){ + inImports = false; + } else { + const match = (line).match(versionRegex); + if(match) { + this.libraries.push({ + name: match[1], + version: parseVersionRange(match[3],match[4]), + type: 'package', + dependencies: [], + getInfo: defaultLibraryGetInfo, + addInfo: defaultLibraryAddInfo + }); + } + } + } + } + console.log('Finished reading the DESCRIPTION-file.'); + } catch(error) { + if(error instanceof Error) { + console.error(`Error reading file: ${error.message}`); + } else { + console.error('Unknown error while reading file.'); + } + } + } +}; \ No newline at end of file diff --git a/src/project/plugins/flowr-analyzer-loading-order-plugin.ts b/src/project/plugins/flowr-analyzer-loading-order-plugin.ts new file mode 100644 index 00000000000..32dcaa6a369 --- /dev/null +++ b/src/project/plugins/flowr-analyzer-loading-order-plugin.ts @@ -0,0 +1,7 @@ +import type { FlowrAnalyzerPlugin } from './flowr-analyzer-plugin'; +import type { PathLike } from 'fs'; + +export interface FlowrAnalyzerLoadingOrderPlugin extends FlowrAnalyzerPlugin { + type: 'loading-order'; + loadingOrder: PathLike[]; +} \ No newline at end of file diff --git a/src/project/plugins/flowr-analyzer-plugin-util.ts b/src/project/plugins/flowr-analyzer-plugin-util.ts new file mode 100644 index 00000000000..e3c8f5aa892 --- /dev/null +++ b/src/project/plugins/flowr-analyzer-plugin-util.ts @@ -0,0 +1,16 @@ +import type { PathLike } from 'fs'; +import { getAllFiles } from '../../util/files'; + +export async function findFileFromRoot(rootPath: PathLike, fileName: string) : Promise { + try { + const files = getAllFiles(rootPath.toString(), new RegExp(`^${fileName}$`)); + const descriptionFilesFound : PathLike[] = []; + for await (const file of files) { + descriptionFilesFound.push(file); + } + return descriptionFilesFound.length > 0 ? descriptionFilesFound[0] : undefined; + } catch(error) { + console.error(`Error reading directory ${rootPath.toString()}:`, error); + } + return undefined; +} \ No newline at end of file diff --git a/src/project/plugins/flowr-analyzer-plugin.ts b/src/project/plugins/flowr-analyzer-plugin.ts new file mode 100644 index 00000000000..17b96feb125 --- /dev/null +++ b/src/project/plugins/flowr-analyzer-plugin.ts @@ -0,0 +1,21 @@ +import type { SemVer } from 'semver'; +import type { FlowrAnalyzer } from '../flowr-analyzer'; +import type { FlowrConfigOptions } from '../../config'; +import type { FlowrAnalyzerLibraryVersionsPlugin } from './flowr-analyzer-library-versions-plugin'; +import type { FlowrAnalyzerLoadingOrderPlugin } from './flowr-analyzer-loading-order-plugin'; +import type { FlowrAnalyzerScopingPlugin } from './flowr-analyzer-scoping-plugin'; + +export interface FlowrAnalyzerPlugin { + name: string; + description: string; + version: SemVer; + type: 'library-versions' | 'loading-order' | 'scoping'; + dependencies?: []; + processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise; +} + +export type AnyFlowrAnalyzerPlugin = + | FlowrAnalyzerLibraryVersionsPlugin + | FlowrAnalyzerLoadingOrderPlugin + | FlowrAnalyzerScopingPlugin; + diff --git a/src/project/plugins/flowr-analyzer-scoping-plugin.ts b/src/project/plugins/flowr-analyzer-scoping-plugin.ts new file mode 100644 index 00000000000..b20ad031fbd --- /dev/null +++ b/src/project/plugins/flowr-analyzer-scoping-plugin.ts @@ -0,0 +1,5 @@ +import type { FlowrAnalyzerPlugin } from './flowr-analyzer-plugin'; + +export interface FlowrAnalyzerScopingPlugin extends FlowrAnalyzerPlugin { + type: 'scoping'; +} \ No newline at end of file From 7953f9a6202059bc7450c8d56dc99990dfcd48ae Mon Sep 17 00:00:00 2001 From: stimjannik Date: Wed, 2 Jul 2025 21:04:32 +0200 Subject: [PATCH 05/70] test(plugin): library versions from DESCRIPTION test --- .../plugin/library-versions-plugin.test.ts | 17 +++++++++ test/testfiles/project/DESCRIPTION | 37 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 test/functionality/project/plugin/library-versions-plugin.test.ts create mode 100644 test/testfiles/project/DESCRIPTION diff --git a/test/functionality/project/plugin/library-versions-plugin.test.ts b/test/functionality/project/plugin/library-versions-plugin.test.ts new file mode 100644 index 00000000000..049474ab5d2 --- /dev/null +++ b/test/functionality/project/plugin/library-versions-plugin.test.ts @@ -0,0 +1,17 @@ +import { describe, assert, test } from 'vitest'; +import { + libraryVersionsDescriptionFilePlugin +} from '../../../../src/project/plugins/flowr-analyzer-library-versions-plugin'; +import path from 'path'; +import type { FlowrAnalyzer } from '../../../../src/project/flowr-analyzer'; +import type { FlowrConfigOptions } from '../../../../src/config'; + +describe('Library-Versions-Plugin', function() { + test('DESCRIPTION-file', async() => { + const plugin = libraryVersionsDescriptionFilePlugin; + plugin.setRootPath(path.resolve('test/testfiles/project')); + await plugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); + console.log(plugin.libraries); + assert.isTrue(plugin.libraries.length === 4); + }); +}); \ No newline at end of file diff --git a/test/testfiles/project/DESCRIPTION b/test/testfiles/project/DESCRIPTION new file mode 100644 index 00000000000..2a5ef89f217 --- /dev/null +++ b/test/testfiles/project/DESCRIPTION @@ -0,0 +1,37 @@ +Package: mypackage +Title: What the Package Does (One Line, Title Case) +Version: 0.0.0.9000 +Authors@R: + person("First", "Last", , "first.last@example.com", role = c("aut", "cre")) +Description: What the package does (one paragraph). +License: `use_mit_license()`, `use_gpl3_license()` or friends to pick a + license +Encoding: UTF-8 +Roxygen: list(markdown = TRUE) +RoxygenNote: 7.3.2 +Description: The description of a package usually spans multiple lines. + The second and subsequent lines should be indented, usually with four + spaces. +URL: https://github.com/flowr-analysis/flowr +BugReports: https://github.com/flowr-analysis/flowr/issues +Depends: + R (>= 4.0) +Imports: + dplyr (>= 1.4.0), + tidyr, + ggplot2 (>= 2.5.8), + ads (>= 0.8.1) +Suggests: + sf (>= 0.8-1), + tibble, + testthat (>= 2.1.5), + svglite (>= 1.1.2), + xml2 + vdiffr (>= 1.5.6), +Enhances: + something +LazyData: true +Collate: + 'aaa.R' + 'main.R' + 'zzz.R' \ No newline at end of file From 16762a90927c21a7dd8dd5f3b60c9578d9c4d1ff Mon Sep 17 00:00:00 2001 From: stimjannik Date: Wed, 2 Jul 2025 21:05:25 +0200 Subject: [PATCH 06/70] feat(plugin): add (un)registering plugins in FlowrAnalyzerBuilder --- src/project/flowr-analyzer-builder.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 2ea29e83f57..3a296bfaccc 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -5,10 +5,12 @@ import type { RParseRequests } from '../r-bridge/retriever'; import { FlowrAnalyzer } from './flowr-analyzer'; import { retrieveEngineInstances } from '../engines'; import type { KnownParser } from '../r-bridge/parser'; +import type { AnyFlowrAnalyzerPlugin } from './plugins/flowr-analyzer-plugin'; export class FlowrAnalyzerBuilder { private flowrConfig: DeepWritable = cloneConfig(defaultConfigOptions); private readonly request: RParseRequests; + private plugins: AnyFlowrAnalyzerPlugin[]; public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions) : this { this.flowrConfig = amendConfig(this.flowrConfig, func); @@ -20,8 +22,25 @@ export class FlowrAnalyzerBuilder { return this; } - constructor(request: RParseRequests) { + constructor(request: RParseRequests, plugins?: AnyFlowrAnalyzerPlugin[]) { this.request = request; + this.plugins = plugins ? plugins : []; + } + + public registerPlugin(plugin: AnyFlowrAnalyzerPlugin | AnyFlowrAnalyzerPlugin[]): void { + if(Array.isArray(plugin)){ + this.plugins.push(...plugin); + } else { + this.plugins.push(plugin); + } + } + + public unregisterPlugin(plugin: AnyFlowrAnalyzerPlugin | AnyFlowrAnalyzerPlugin[]): void { + if(Array.isArray(plugin)){ + this.plugins = this.plugins.filter(p => !plugin.includes(p)); + } else { + this.plugins = this.plugins.filter(p => p !== plugin); + } } public async build(): Promise { From ad9d9622f944cdc9443f60777c650ed046dcbe47 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:29:33 +0200 Subject: [PATCH 07/70] feat(builder): simplifications --- src/project/flowr-analyzer-builder.ts | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 3a296bfaccc..e2efe8478e9 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -24,23 +24,17 @@ export class FlowrAnalyzerBuilder { constructor(request: RParseRequests, plugins?: AnyFlowrAnalyzerPlugin[]) { this.request = request; - this.plugins = plugins ? plugins : []; + this.plugins = plugins ?? []; } - public registerPlugin(plugin: AnyFlowrAnalyzerPlugin | AnyFlowrAnalyzerPlugin[]): void { - if(Array.isArray(plugin)){ - this.plugins.push(...plugin); - } else { - this.plugins.push(plugin); - } + public registerPlugin(plugin: [...AnyFlowrAnalyzerPlugin[]]): this { + this.plugins.push(...plugin); + return this; } - public unregisterPlugin(plugin: AnyFlowrAnalyzerPlugin | AnyFlowrAnalyzerPlugin[]): void { - if(Array.isArray(plugin)){ - this.plugins = this.plugins.filter(p => !plugin.includes(p)); - } else { - this.plugins = this.plugins.filter(p => p !== plugin); - } + public unregisterPlugin(plugin: [...AnyFlowrAnalyzerPlugin[]]): this { + this.plugins = this.plugins.filter(p => !plugin.includes(p)); + return this; } public async build(): Promise { From 13841b507e33b8cb71e2af41510379d43ede0feb Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:31:55 +0200 Subject: [PATCH 08/70] feat(builder): use EngineConfig type --- src/project/flowr-analyzer-builder.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index e2efe8478e9..15f0f74ec55 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -1,4 +1,4 @@ -import type { FlowrConfigOptions } from '../config'; +import type { EngineConfig, FlowrConfigOptions } from '../config'; import { amendConfig, cloneConfig, defaultConfigOptions } from '../config'; import type { DeepWritable } from 'ts-essentials'; import type { RParseRequests } from '../r-bridge/retriever'; @@ -17,7 +17,7 @@ export class FlowrAnalyzerBuilder { return this; } - public setEngine(engine : 'tree-sitter' | 'r-shell') { + public setEngine(engine : EngineConfig['type']) { this.flowrConfig.defaultEngine = engine; return this; } From 17ac95603dd4221564387bbdc39d833961e2618a Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Thu, 17 Jul 2025 11:26:01 +0200 Subject: [PATCH 09/70] feat(builder): use varargs/rest parameter syntax --- src/project/flowr-analyzer-builder.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 15f0f74ec55..0a963994526 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -27,12 +27,12 @@ export class FlowrAnalyzerBuilder { this.plugins = plugins ?? []; } - public registerPlugin(plugin: [...AnyFlowrAnalyzerPlugin[]]): this { + public registerPlugin(...plugin: AnyFlowrAnalyzerPlugin[]): this { this.plugins.push(...plugin); return this; } - public unregisterPlugin(plugin: [...AnyFlowrAnalyzerPlugin[]]): this { + public unregisterPlugin(...plugin: AnyFlowrAnalyzerPlugin[]): this { this.plugins = this.plugins.filter(p => !plugin.includes(p)); return this; } From 7ea5a3133fc97d6c122fee34a92cbc89684907ee Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 17 Jul 2025 14:38:28 +0200 Subject: [PATCH 10/70] refactor(plugin): use abstract classes instead of interfaces only --- .../flowr-analyzer-description-file-plugin.ts | 22 +++ .../flowr-analyzer-file-plugin.ts | 7 + .../flowr-analyzer-library-versions-plugin.ts | 152 ------------------ .../plugins/flowr-analyzer-plugin-util.ts | 16 -- src/project/plugins/flowr-analyzer-plugin.ts | 39 +++-- .../flowr-analyzer-loading-order-plugin.ts | 2 +- ...ackage-versions-description-file-plugin.ts | 18 +++ .../flowr-analyzer-package-versions-plugin.ts | 7 + .../package-version-plugins/package.ts | 44 +++++ .../flowr-analyzer-scoping-plugin.ts | 2 +- 10 files changed, 125 insertions(+), 184 deletions(-) create mode 100644 src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts create mode 100644 src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts delete mode 100644 src/project/plugins/flowr-analyzer-library-versions-plugin.ts delete mode 100644 src/project/plugins/flowr-analyzer-plugin-util.ts rename src/project/plugins/{ => loading-order-plugins}/flowr-analyzer-loading-order-plugin.ts (72%) create mode 100644 src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts create mode 100644 src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-plugin.ts create mode 100644 src/project/plugins/package-version-plugins/package.ts rename src/project/plugins/{ => scoping-plugins}/flowr-analyzer-scoping-plugin.ts (58%) diff --git a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts new file mode 100644 index 00000000000..97f724e8df4 --- /dev/null +++ b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts @@ -0,0 +1,22 @@ +import { FlowrAnalyzerFilePlugin } from './flowr-analyzer-file-plugin'; +import { SemVer } from 'semver'; +import type { FlowrAnalyzer } from '../../flowr-analyzer'; +import type { FlowrConfigOptions } from '../../../config'; +import type { Package } from '../package-version-plugins/package'; +import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; + +export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin{ + public readonly name = 'flowr-analyzer-description-file-plugin'; + public readonly description = 'This plugin does...'; + public readonly version = new SemVer('0.1.0'); + + private content: Map = new Map(); + + packages: Package[] = []; + dependencies: FlowrAnalyzerPlugin[] = []; + + public async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { + await fetch(''); + this.content.set('',''); + } +} \ No newline at end of file diff --git a/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts new file mode 100644 index 00000000000..489a37e93a9 --- /dev/null +++ b/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts @@ -0,0 +1,7 @@ +import { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; +import type { PathLike } from 'fs'; + +export abstract class FlowrAnalyzerFilePlugin extends FlowrAnalyzerPlugin { + public readonly type = 'file'; + protected files: PathLike[] | undefined = []; +} \ No newline at end of file diff --git a/src/project/plugins/flowr-analyzer-library-versions-plugin.ts b/src/project/plugins/flowr-analyzer-library-versions-plugin.ts deleted file mode 100644 index fe392d23a36..00000000000 --- a/src/project/plugins/flowr-analyzer-library-versions-plugin.ts +++ /dev/null @@ -1,152 +0,0 @@ -import type { FlowrAnalyzerPlugin } from './flowr-analyzer-plugin'; -import { Range , SemVer } from 'semver'; -import type { FlowrAnalyzer } from '../flowr-analyzer'; -import type { FlowrConfigOptions } from '../../config'; -import type { PathLike } from 'fs'; -import fs from 'fs'; -import readline from 'readline'; -import { findFileFromRoot } from './flowr-analyzer-plugin-util'; - -export interface FlowrAnalyzerLibraryVersionsPlugin extends FlowrAnalyzerPlugin { - type: 'library-versions'; - libraries: Library[]; -} - -interface Library { - name: string; - version?: Range; - dependencies?: Library[]; - type: 'package' | 'system' | 'r'; - addInfo(name?: string, versionConstraint?: Range, type?: 'package' | 'system' | 'r', dependency?: []): void; - getInfo(): Library; -} - -function defaultLibraryAddInfo(this: Library, name?: string, versionConstraint?: Range, type?: 'package' | 'system' | 'r', dependency?: []): void { - if(name){ - this.name = name; - } - if(versionConstraint){ - this.version = versionConstraint; - } - if(type){ - this.type = type; - } - if(dependency){ - if(!this.dependencies){ - this.dependencies = dependency; - } else { - if(Array.isArray(dependency)){ - this.dependencies.push(...dependency); - } else { - this.dependencies.push(dependency); - } - } - } -} - -function defaultLibraryGetInfo(this: Library): Library { - return this; -} - -function parseVersionRange(constraint?: string, version?: string): Range | undefined { - if(constraint && version) { - return new Range(constraint + version); - } - if(version) { - return new Range(version); - } - return undefined; -} - -export async function findFilesForPlugin(files: PathLike | PathLike[] | undefined, findFilesMethod: (rootPath: PathLike) => Promise, rootPath: PathLike) : Promise { - if(files === undefined){ - const foundFiles: PathLike | PathLike[] | undefined = await findFilesMethod(rootPath); - if(foundFiles === undefined) { - console.log('Could not locate DESCRIPTION file!'); - return undefined; - } - return foundFiles; - } - return files; -} - -export const libraryVersionsDescriptionFilePlugin: FlowrAnalyzerLibraryVersionsPlugin & { - descriptionFile: PathLike | undefined; - rootPath: PathLike | undefined; - findDescriptionFile: (rootPath: PathLike) => Promise; - setDescriptionFile: (descriptionFile: PathLike) => void; - setRootPath: (rootpath: PathLike) => void; -} = { - name: 'library-versions-description-file', - description: 'This plugin does...', - version: new SemVer('0.1.0'), - type: 'library-versions', - libraries: [], - descriptionFile: undefined, - rootPath: undefined, - setRootPath(rootPath: PathLike): void { - this.rootPath = rootPath; - }, - async findDescriptionFile(rootPath: PathLike): Promise{ - return findFileFromRoot(rootPath, 'DESCRIPTION'); - }, - setDescriptionFile(descriptionFile: PathLike){ - this.descriptionFile = descriptionFile; - }, - async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { - console.log('Running Description-file Library Versions Plugin...'); - console.log(analyzer); - console.log(pluginConfig); - - this.descriptionFile = await findFilesForPlugin(this.descriptionFile, this.findDescriptionFile, this.rootPath ? this.rootPath : '') as PathLike | undefined; - - if(this.descriptionFile === undefined){ - return; - } - - try { - const readStream = fs.createReadStream(this.descriptionFile, { encoding: 'utf8' }); - const rl = readline.createInterface({ - input: readStream, - crlfDelay: Infinity - }); - console.log('Reading DESCRIPTION file...'); - - let inImports: boolean = false; - const importsRegex = new RegExp('^Imports:\\s*$'); - const versionRegex = new RegExp('^\\s*([a-zA-Z0-9._-]+)\\s*(\\((>=|<=|>|<|=)\\s*([0-9A-Za-z\\-+.*]+)\\))?,?\\s*$'); - const indentRegex = new RegExp('^\\s+'); - - for await (const line of rl) { - if(importsRegex.test(line)){ - inImports = true; - continue; - } - if(inImports){ - if(!indentRegex.test(line)){ - inImports = false; - } else { - const match = (line).match(versionRegex); - if(match) { - this.libraries.push({ - name: match[1], - version: parseVersionRange(match[3],match[4]), - type: 'package', - dependencies: [], - getInfo: defaultLibraryGetInfo, - addInfo: defaultLibraryAddInfo - }); - } - } - } - } - console.log('Finished reading the DESCRIPTION-file.'); - } catch(error) { - if(error instanceof Error) { - console.error(`Error reading file: ${error.message}`); - } else { - console.error('Unknown error while reading file.'); - } - } - } -}; \ No newline at end of file diff --git a/src/project/plugins/flowr-analyzer-plugin-util.ts b/src/project/plugins/flowr-analyzer-plugin-util.ts deleted file mode 100644 index e3c8f5aa892..00000000000 --- a/src/project/plugins/flowr-analyzer-plugin-util.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { PathLike } from 'fs'; -import { getAllFiles } from '../../util/files'; - -export async function findFileFromRoot(rootPath: PathLike, fileName: string) : Promise { - try { - const files = getAllFiles(rootPath.toString(), new RegExp(`^${fileName}$`)); - const descriptionFilesFound : PathLike[] = []; - for await (const file of files) { - descriptionFilesFound.push(file); - } - return descriptionFilesFound.length > 0 ? descriptionFilesFound[0] : undefined; - } catch(error) { - console.error(`Error reading directory ${rootPath.toString()}:`, error); - } - return undefined; -} \ No newline at end of file diff --git a/src/project/plugins/flowr-analyzer-plugin.ts b/src/project/plugins/flowr-analyzer-plugin.ts index 17b96feb125..6d493a389a5 100644 --- a/src/project/plugins/flowr-analyzer-plugin.ts +++ b/src/project/plugins/flowr-analyzer-plugin.ts @@ -1,21 +1,32 @@ import type { SemVer } from 'semver'; -import type { FlowrAnalyzer } from '../flowr-analyzer'; +import type { FlowrAnalyzer } from '../flowr-analyzer'; import type { FlowrConfigOptions } from '../../config'; -import type { FlowrAnalyzerLibraryVersionsPlugin } from './flowr-analyzer-library-versions-plugin'; -import type { FlowrAnalyzerLoadingOrderPlugin } from './flowr-analyzer-loading-order-plugin'; -import type { FlowrAnalyzerScopingPlugin } from './flowr-analyzer-scoping-plugin'; +import type { PathLike } from 'fs'; + +export type PluginType = 'package-versions' | 'loading-order' | 'scoping' | 'file'; + +export interface FlowrAnalyzerPluginInterface { + readonly name: string; + readonly description: string; + readonly version: SemVer; + readonly type: PluginType; + dependencies: FlowrAnalyzerPlugin[]; -export interface FlowrAnalyzerPlugin { - name: string; - description: string; - version: SemVer; - type: 'library-versions' | 'loading-order' | 'scoping'; - dependencies?: []; processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise; } -export type AnyFlowrAnalyzerPlugin = - | FlowrAnalyzerLibraryVersionsPlugin - | FlowrAnalyzerLoadingOrderPlugin - | FlowrAnalyzerScopingPlugin; +export abstract class FlowrAnalyzerPlugin implements FlowrAnalyzerPluginInterface { + public abstract readonly name: string; + public abstract readonly description: string; + public abstract readonly version: SemVer; + public abstract readonly type: PluginType; + public abstract dependencies: FlowrAnalyzerPlugin[]; + public rootPath: PathLike | undefined; + + public setRootPath(rootPath: PathLike): void { + this.rootPath = rootPath; + } + + public abstract processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise; +} diff --git a/src/project/plugins/flowr-analyzer-loading-order-plugin.ts b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-plugin.ts similarity index 72% rename from src/project/plugins/flowr-analyzer-loading-order-plugin.ts rename to src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-plugin.ts index 32dcaa6a369..6bdd9e11be4 100644 --- a/src/project/plugins/flowr-analyzer-loading-order-plugin.ts +++ b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-plugin.ts @@ -1,4 +1,4 @@ -import type { FlowrAnalyzerPlugin } from './flowr-analyzer-plugin'; +import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import type { PathLike } from 'fs'; export interface FlowrAnalyzerLoadingOrderPlugin extends FlowrAnalyzerPlugin { diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts new file mode 100644 index 00000000000..faf2a617a26 --- /dev/null +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts @@ -0,0 +1,18 @@ +import { FlowrAnalyzerPackageVersionsPlugin } from './flowr-analyzer-package-versions-plugin'; +import { FlowrAnalyzerDescriptionFilePlugin } from '../file-plugins/flowr-analyzer-description-file-plugin'; +import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; +import { SemVer } from 'semver'; +import type { FlowrAnalyzer } from '../../flowr-analyzer'; +import type { FlowrConfigOptions } from '../../../config'; + +class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnalyzerPackageVersionsPlugin { + public readonly name = 'flowr-analyzer-package-version-description-file-plugin'; + public readonly description = 'This plugin does...'; + public readonly version = new SemVer('0.1.0'); + + dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; + + processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { + return Promise.resolve(undefined); + } +} \ No newline at end of file diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-plugin.ts new file mode 100644 index 00000000000..096978d6ea3 --- /dev/null +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-plugin.ts @@ -0,0 +1,7 @@ +import type { Package } from './package'; +import { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; + +export abstract class FlowrAnalyzerPackageVersionsPlugin extends FlowrAnalyzerPlugin { + readonly type = 'package-versions'; + public packages: Package[] = []; +} \ No newline at end of file diff --git a/src/project/plugins/package-version-plugins/package.ts b/src/project/plugins/package-version-plugins/package.ts new file mode 100644 index 00000000000..87798c2def7 --- /dev/null +++ b/src/project/plugins/package-version-plugins/package.ts @@ -0,0 +1,44 @@ +import { Range } from 'semver'; + +type PackageType = 'package' | 'system' | 'r'; + +export class Package { + public name: string; + public version?: Range; + public dependencies?: Package[]; + public type: PackageType; + + public constructor(name: string, type: PackageType, version: Range | undefined, dependencies?: Package[]) { + this.name = name; + this.type = type; + this.version = version; + this.dependencies = dependencies; + } + + public addInfo(name?: string, versionConstraint?: Range, type?: PackageType, dependencies?: Package[]): void { + if(name !== undefined) { + this.name = name; + } + if(versionConstraint !== undefined) { + this.version = versionConstraint; + } + if(type !== undefined) { + this.type = type; + } + if(dependencies !== undefined) { + this.dependencies = dependencies; + } + } + + public getInfo(): this { + return this; + } + + public static parsePackageVersionRange(constraint?: string, version?: string): Range | undefined { + if(version) { + return constraint ? new Range(constraint + version) : new Range(version); + } else { + return undefined; + } + } +} \ No newline at end of file diff --git a/src/project/plugins/flowr-analyzer-scoping-plugin.ts b/src/project/plugins/scoping-plugins/flowr-analyzer-scoping-plugin.ts similarity index 58% rename from src/project/plugins/flowr-analyzer-scoping-plugin.ts rename to src/project/plugins/scoping-plugins/flowr-analyzer-scoping-plugin.ts index b20ad031fbd..5939ee5ec9c 100644 --- a/src/project/plugins/flowr-analyzer-scoping-plugin.ts +++ b/src/project/plugins/scoping-plugins/flowr-analyzer-scoping-plugin.ts @@ -1,4 +1,4 @@ -import type { FlowrAnalyzerPlugin } from './flowr-analyzer-plugin'; +import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; export interface FlowrAnalyzerScopingPlugin extends FlowrAnalyzerPlugin { type: 'scoping'; From 4d097e623468c5e40961572437fdedd5c484f0de Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 17 Jul 2025 14:58:05 +0200 Subject: [PATCH 11/70] lint-fix(plugin): fixed linter issues --- src/project/flowr-analyzer-builder.ts | 10 +++++----- src/project/plugins/flowr-analyzer-plugin.ts | 2 +- ...ackage-versions-description-file-plugin.ts | 4 +++- .../plugin/library-versions-plugin.test.ts | 19 ++++++++----------- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 0a963994526..870ce41ae0e 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -5,12 +5,12 @@ import type { RParseRequests } from '../r-bridge/retriever'; import { FlowrAnalyzer } from './flowr-analyzer'; import { retrieveEngineInstances } from '../engines'; import type { KnownParser } from '../r-bridge/parser'; -import type { AnyFlowrAnalyzerPlugin } from './plugins/flowr-analyzer-plugin'; +import type { FlowrAnalyzerPlugin } from './plugins/flowr-analyzer-plugin'; export class FlowrAnalyzerBuilder { private flowrConfig: DeepWritable = cloneConfig(defaultConfigOptions); private readonly request: RParseRequests; - private plugins: AnyFlowrAnalyzerPlugin[]; + private plugins: FlowrAnalyzerPlugin[]; public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions) : this { this.flowrConfig = amendConfig(this.flowrConfig, func); @@ -22,17 +22,17 @@ export class FlowrAnalyzerBuilder { return this; } - constructor(request: RParseRequests, plugins?: AnyFlowrAnalyzerPlugin[]) { + constructor(request: RParseRequests, plugins?: FlowrAnalyzerPlugin[]) { this.request = request; this.plugins = plugins ?? []; } - public registerPlugin(...plugin: AnyFlowrAnalyzerPlugin[]): this { + public registerPlugin(...plugin: FlowrAnalyzerPlugin[]): this { this.plugins.push(...plugin); return this; } - public unregisterPlugin(...plugin: AnyFlowrAnalyzerPlugin[]): this { + public unregisterPlugin(...plugin: FlowrAnalyzerPlugin[]): this { this.plugins = this.plugins.filter(p => !plugin.includes(p)); return this; } diff --git a/src/project/plugins/flowr-analyzer-plugin.ts b/src/project/plugins/flowr-analyzer-plugin.ts index 6d493a389a5..26a4796eaaf 100644 --- a/src/project/plugins/flowr-analyzer-plugin.ts +++ b/src/project/plugins/flowr-analyzer-plugin.ts @@ -23,7 +23,7 @@ export abstract class FlowrAnalyzerPlugin implements FlowrAnalyzerPluginInterfac public abstract dependencies: FlowrAnalyzerPlugin[]; public rootPath: PathLike | undefined; - public setRootPath(rootPath: PathLike): void { + public setRootPath(rootPath: PathLike | undefined): void { this.rootPath = rootPath; } diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts index faf2a617a26..e739f2c1ac9 100644 --- a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts @@ -5,7 +5,7 @@ import { SemVer } from 'semver'; import type { FlowrAnalyzer } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; -class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnalyzerPackageVersionsPlugin { +export class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnalyzerPackageVersionsPlugin { public readonly name = 'flowr-analyzer-package-version-description-file-plugin'; public readonly description = 'This plugin does...'; public readonly version = new SemVer('0.1.0'); @@ -13,6 +13,8 @@ class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnalyzerPac dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { + console.log(analyzer); + console.log(pluginConfig); return Promise.resolve(undefined); } } \ No newline at end of file diff --git a/test/functionality/project/plugin/library-versions-plugin.test.ts b/test/functionality/project/plugin/library-versions-plugin.test.ts index 049474ab5d2..5bc8821ba6a 100644 --- a/test/functionality/project/plugin/library-versions-plugin.test.ts +++ b/test/functionality/project/plugin/library-versions-plugin.test.ts @@ -1,17 +1,14 @@ import { describe, assert, test } from 'vitest'; -import { - libraryVersionsDescriptionFilePlugin -} from '../../../../src/project/plugins/flowr-analyzer-library-versions-plugin'; -import path from 'path'; -import type { FlowrAnalyzer } from '../../../../src/project/flowr-analyzer'; -import type { FlowrConfigOptions } from '../../../../src/config'; + + describe('Library-Versions-Plugin', function() { test('DESCRIPTION-file', async() => { - const plugin = libraryVersionsDescriptionFilePlugin; - plugin.setRootPath(path.resolve('test/testfiles/project')); - await plugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); - console.log(plugin.libraries); - assert.isTrue(plugin.libraries.length === 4); + // const plugin = libraryVersionsDescriptionFilePlugin; + // plugin.setRootPath(path.resolve('test/testfiles/project')); + // await plugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); + // console.log(plugin.libraries); + // assert.isTrue(plugin.libraries.length === 4); + assert.isTrue(true); }); }); \ No newline at end of file From 54f6612cad02e51505d17d16b0cbb7fca4b66417 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 17 Jul 2025 15:25:41 +0200 Subject: [PATCH 12/70] lint-fix(plugin): fixed linter issues --- .../flowr-analyzer-description-file-plugin.ts | 4 +++- .../project/plugin/library-versions-plugin.test.ts | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts index 97f724e8df4..8eb6c9646aa 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts @@ -16,7 +16,9 @@ export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin{ dependencies: FlowrAnalyzerPlugin[] = []; public async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { - await fetch(''); + await Promise.resolve(); this.content.set('',''); + console.log(analyzer); + console.log(pluginConfig); } } \ No newline at end of file diff --git a/test/functionality/project/plugin/library-versions-plugin.test.ts b/test/functionality/project/plugin/library-versions-plugin.test.ts index 5bc8821ba6a..1bcbde69cf2 100644 --- a/test/functionality/project/plugin/library-versions-plugin.test.ts +++ b/test/functionality/project/plugin/library-versions-plugin.test.ts @@ -1,12 +1,17 @@ import { describe, assert, test } from 'vitest'; +import { + FlowrAnalyzerDescriptionFilePlugin +} from '../../../../src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin'; +import type { FlowrAnalyzer } from '../../../../src/project/flowr-analyzer'; +import type { FlowrConfigOptions } from '../../../../src/config'; describe('Library-Versions-Plugin', function() { test('DESCRIPTION-file', async() => { - // const plugin = libraryVersionsDescriptionFilePlugin; + const plugin = new FlowrAnalyzerDescriptionFilePlugin(); // plugin.setRootPath(path.resolve('test/testfiles/project')); - // await plugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); + await plugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); // console.log(plugin.libraries); // assert.isTrue(plugin.libraries.length === 4); assert.isTrue(true); From 39e7ba72cb585d0724d076c4d7bd946559bd16ab Mon Sep 17 00:00:00 2001 From: stimjannik Date: Tue, 22 Jul 2025 15:44:30 +0200 Subject: [PATCH 13/70] feat(plugin): add new DESCRIPTION parser --- .../flowr-analyzer-description-file-plugin.ts | 60 +++++++++++++++---- .../flowr-analyzer-file-plugin.ts | 6 +- ...ackage-versions-description-file-plugin.ts | 30 +++++++++- .../package-version-plugins/package.ts | 13 ++-- .../plugin/library-versions-plugin.test.ts | 19 ------ 5 files changed, 87 insertions(+), 41 deletions(-) delete mode 100644 test/functionality/project/plugin/library-versions-plugin.test.ts diff --git a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts index 8eb6c9646aa..c68066c5382 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts @@ -1,24 +1,62 @@ import { FlowrAnalyzerFilePlugin } from './flowr-analyzer-file-plugin'; import { SemVer } from 'semver'; -import type { FlowrAnalyzer } from '../../flowr-analyzer'; +import type { FlowrAnalyzer } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; -import type { Package } from '../package-version-plugins/package'; -import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; +import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; +import { readLineByLine } from '../../../util/files'; +import type { PathLike } from 'fs'; +import * as console from 'node:console'; -export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin{ +export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin { public readonly name = 'flowr-analyzer-description-file-plugin'; public readonly description = 'This plugin does...'; public readonly version = new SemVer('0.1.0'); - - private content: Map = new Map(); - - packages: Package[] = []; - dependencies: FlowrAnalyzerPlugin[] = []; + public readonly dependencies: FlowrAnalyzerPlugin[] = []; + public readonly information: Map = new Map(); public async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { - await Promise.resolve(); - this.content.set('',''); + for(const file of this.files) { + const parsedDescriptionFile = await this.parseDescription(file); + for(const [key, values] of parsedDescriptionFile) { + this.information.set(key, values); + } + } + console.log(analyzer); console.log(pluginConfig); } + + private async parseDescription(file: PathLike): Promise> { + const result = new Map(); + let currentKey = ''; + let currentValue = ''; + + await readLineByLine(file.toString(), async(lineBuf) => { + const line = lineBuf.toString(); + + if(/^\s/.test(line)) { + currentValue += '\n' + line.trim(); + } else { + if(currentKey) { + const values = currentValue + ? currentValue.split(/[\n,]+/).map(s => s.trim().replace(/'/g, '')).filter(s => s.length > 0) + : []; + result.set(currentKey, values); + } + + const [key, ...rest] = line.split(':'); + currentKey = key.trim(); + currentValue = rest.join(':').trim(); + } + }); + + if(currentKey) { + const values = currentValue + ? currentValue.split(/[\n,]+/).map(s => s.trim().replace(/'/g, '')).filter(s => s.length > 0) + : []; + result.set(currentKey, values); + } + + return result; + } } \ No newline at end of file diff --git a/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts index 489a37e93a9..1b0afb4e159 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts @@ -3,5 +3,9 @@ import type { PathLike } from 'fs'; export abstract class FlowrAnalyzerFilePlugin extends FlowrAnalyzerPlugin { public readonly type = 'file'; - protected files: PathLike[] | undefined = []; + protected files: PathLike[] = []; + + public addFiles(...files: PathLike[]) : void { + this.files.push(...files); + } } \ No newline at end of file diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts index e739f2c1ac9..21645f34ad1 100644 --- a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts @@ -4,17 +4,43 @@ import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import { SemVer } from 'semver'; import type { FlowrAnalyzer } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; +import type { PackageType } from './package'; +import { Package } from './package'; export class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnalyzerPackageVersionsPlugin { public readonly name = 'flowr-analyzer-package-version-description-file-plugin'; public readonly description = 'This plugin does...'; public readonly version = new SemVer('0.1.0'); - dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; + dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; + descriptionFile: Map = new Map(); + + async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { + const plugin = this.dependencies[0] as FlowrAnalyzerDescriptionFilePlugin; + await plugin.processor(analyzer, pluginConfig); + this.descriptionFile = plugin.information; + + this.retrieveVersionsFromField('Depends', 'r'); + this.retrieveVersionsFromField('Imports', 'package'); - processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { console.log(analyzer); console.log(pluginConfig); return Promise.resolve(undefined); } + + private retrieveVersionsFromField(field: string, type?: PackageType) : void{ + for(const entry of this.descriptionFile?.get(field) || []) { + const match = RegExp(/^([a-zA-Z0-9.]+)(?:\s*\(([><=~!]+)\s*([\d.]+)\))?$/).exec(entry); + + if(match) { + const name = match[1]; + const operator = match[2]; + const version = match[3]; + + const range = Package.parsePackageVersionRange(operator, version); + + this.packages.push(new Package(name, range, type)); + } + } + } } \ No newline at end of file diff --git a/src/project/plugins/package-version-plugins/package.ts b/src/project/plugins/package-version-plugins/package.ts index 87798c2def7..8de31f28273 100644 --- a/src/project/plugins/package-version-plugins/package.ts +++ b/src/project/plugins/package-version-plugins/package.ts @@ -1,18 +1,15 @@ import { Range } from 'semver'; -type PackageType = 'package' | 'system' | 'r'; +export type PackageType = 'package' | 'system' | 'r'; export class Package { - public name: string; + public name?: string; public version?: Range; + public type?: PackageType; public dependencies?: Package[]; - public type: PackageType; - public constructor(name: string, type: PackageType, version: Range | undefined, dependencies?: Package[]) { - this.name = name; - this.type = type; - this.version = version; - this.dependencies = dependencies; + constructor(name?: string, version?: Range, type?: PackageType, dependencies?: Package[]) { + this.addInfo(name, version, type, dependencies); } public addInfo(name?: string, versionConstraint?: Range, type?: PackageType, dependencies?: Package[]): void { diff --git a/test/functionality/project/plugin/library-versions-plugin.test.ts b/test/functionality/project/plugin/library-versions-plugin.test.ts deleted file mode 100644 index 1bcbde69cf2..00000000000 --- a/test/functionality/project/plugin/library-versions-plugin.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, assert, test } from 'vitest'; -import { - FlowrAnalyzerDescriptionFilePlugin -} from '../../../../src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin'; -import type { FlowrAnalyzer } from '../../../../src/project/flowr-analyzer'; -import type { FlowrConfigOptions } from '../../../../src/config'; - - - -describe('Library-Versions-Plugin', function() { - test('DESCRIPTION-file', async() => { - const plugin = new FlowrAnalyzerDescriptionFilePlugin(); - // plugin.setRootPath(path.resolve('test/testfiles/project')); - await plugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); - // console.log(plugin.libraries); - // assert.isTrue(plugin.libraries.length === 4); - assert.isTrue(true); - }); -}); \ No newline at end of file From 62e50ae960bc21916461479877fcc272b6c443b9 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Tue, 22 Jul 2025 15:45:20 +0200 Subject: [PATCH 14/70] feat(plugin): add loading order from DESCRIPTION retriever --- ...r-loading-order-description-file-plugin.ts | 27 +++++++++++++++++++ .../flowr-analyzer-loading-order-plugin.ts | 8 +++--- 2 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts diff --git a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts new file mode 100644 index 00000000000..3bc87f23aed --- /dev/null +++ b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts @@ -0,0 +1,27 @@ +import { FlowrAnalyzerDescriptionFilePlugin } from '../file-plugins/flowr-analyzer-description-file-plugin'; +import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; +import { SemVer } from 'semver'; +import type { FlowrAnalyzer } from '../../flowr-analyzer'; +import type { FlowrConfigOptions } from '../../../config'; +import { FlowrAnalyzerLoadingOrderPlugin } from './flowr-analyzer-loading-order-plugin'; + +export class FlowrAnalyzerLoadingOrderDescriptionFilePlugin extends FlowrAnalyzerLoadingOrderPlugin { + public readonly name = 'flowr-analyzer-package-version-description-file-plugin'; + public readonly description = 'This plugin does...'; + public readonly version = new SemVer('0.1.0'); + + dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; + descriptionFile: Map = new Map(); + + async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { + const plugin = this.dependencies[0] as FlowrAnalyzerDescriptionFilePlugin; + await plugin.processor(analyzer, pluginConfig); + this.descriptionFile = plugin.information; + + this.loadingOrder = this.descriptionFile?.get('Collate')?.slice() || []; + + console.log(analyzer); + console.log(pluginConfig); + return Promise.resolve(undefined); + } +} \ No newline at end of file diff --git a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-plugin.ts b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-plugin.ts index 6bdd9e11be4..8e91625249e 100644 --- a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-plugin.ts +++ b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-plugin.ts @@ -1,7 +1,7 @@ -import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; +import { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import type { PathLike } from 'fs'; -export interface FlowrAnalyzerLoadingOrderPlugin extends FlowrAnalyzerPlugin { - type: 'loading-order'; - loadingOrder: PathLike[]; +export abstract class FlowrAnalyzerLoadingOrderPlugin extends FlowrAnalyzerPlugin { + readonly type = 'loading-order'; + public loadingOrder: PathLike[] = []; } \ No newline at end of file From 809670c07cc2adfb15b6a9c6337190c9a17a017e Mon Sep 17 00:00:00 2001 From: stimjannik Date: Tue, 22 Jul 2025 15:58:25 +0200 Subject: [PATCH 15/70] test-fix(plugin): adjust description file test --- .../project/plugin/description-file.test.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 test/functionality/project/plugin/description-file.test.ts diff --git a/test/functionality/project/plugin/description-file.test.ts b/test/functionality/project/plugin/description-file.test.ts new file mode 100644 index 00000000000..38bf00e2eb6 --- /dev/null +++ b/test/functionality/project/plugin/description-file.test.ts @@ -0,0 +1,40 @@ +import { describe, assert, test } from 'vitest'; +import { + FlowrAnalyzerDescriptionFilePlugin +} from '../../../../src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin'; +import type { FlowrAnalyzer } from '../../../../src/project/flowr-analyzer'; +import type { FlowrConfigOptions } from '../../../../src/config'; +import path from 'path'; +import { + FlowrAnalyzerPackageVersionsDescriptionFilePlugin +} from '../../../../src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin'; +import { + FlowrAnalyzerLoadingOrderDescriptionFilePlugin +} from '../../../../src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin'; + + +describe('DESCRIPTION-file', function() { + const descriptionFilePlugin = new FlowrAnalyzerDescriptionFilePlugin(); + descriptionFilePlugin.addFiles(path.resolve('test/testfiles/project/DESCRIPTION')); + describe.sequential('Parsing', function() { + test('Library-Versions-Plugin', async() => { + const flowrAnalyzerPackageVersionsDescriptionFilePlugin = new FlowrAnalyzerPackageVersionsDescriptionFilePlugin(); + flowrAnalyzerPackageVersionsDescriptionFilePlugin.dependencies = [descriptionFilePlugin]; + + await flowrAnalyzerPackageVersionsDescriptionFilePlugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); + + console.log(flowrAnalyzerPackageVersionsDescriptionFilePlugin.packages); + assert.isNotEmpty(flowrAnalyzerPackageVersionsDescriptionFilePlugin.packages); + }); + + test('Loading-Order-Plugin', async() => { + const flowrAnalyzerLoadingOrderDescriptionFilePlugin = new FlowrAnalyzerLoadingOrderDescriptionFilePlugin(); + flowrAnalyzerLoadingOrderDescriptionFilePlugin.dependencies = [descriptionFilePlugin]; + + await flowrAnalyzerLoadingOrderDescriptionFilePlugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); + + console.log(flowrAnalyzerLoadingOrderDescriptionFilePlugin.loadingOrder); + assert.isNotEmpty(flowrAnalyzerLoadingOrderDescriptionFilePlugin.loadingOrder); + }); + }); +}); \ No newline at end of file From 6995247a4faf46a88c4ecd065e5c290e59ab74f4 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Tue, 22 Jul 2025 15:59:30 +0200 Subject: [PATCH 16/70] lint(plugin): adjust analyzer and pluginConfig to be ignored --- .../file-plugins/flowr-analyzer-description-file-plugin.ts | 7 ++----- ...flowr-analyzer-loading-order-description-file-plugin.ts | 2 -- ...wr-analyzer-package-versions-description-file-plugin.ts | 2 -- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts index c68066c5382..f1d387924f9 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts @@ -5,7 +5,6 @@ import type { FlowrConfigOptions } from '../../../config'; import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import { readLineByLine } from '../../../util/files'; import type { PathLike } from 'fs'; -import * as console from 'node:console'; export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin { public readonly name = 'flowr-analyzer-description-file-plugin'; @@ -14,16 +13,13 @@ export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin public readonly dependencies: FlowrAnalyzerPlugin[] = []; public readonly information: Map = new Map(); - public async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { + public async processor(_analyzer: FlowrAnalyzer, _pluginConfig: FlowrConfigOptions): Promise { for(const file of this.files) { const parsedDescriptionFile = await this.parseDescription(file); for(const [key, values] of parsedDescriptionFile) { this.information.set(key, values); } } - - console.log(analyzer); - console.log(pluginConfig); } private async parseDescription(file: PathLike): Promise> { @@ -31,6 +27,7 @@ export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin let currentKey = ''; let currentValue = ''; + // eslint-disable-next-line @typescript-eslint/require-await await readLineByLine(file.toString(), async(lineBuf) => { const line = lineBuf.toString(); diff --git a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts index 3bc87f23aed..be93e6105e1 100644 --- a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts +++ b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts @@ -20,8 +20,6 @@ export class FlowrAnalyzerLoadingOrderDescriptionFilePlugin extends FlowrAnalyze this.loadingOrder = this.descriptionFile?.get('Collate')?.slice() || []; - console.log(analyzer); - console.log(pluginConfig); return Promise.resolve(undefined); } } \ No newline at end of file diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts index 21645f34ad1..3ec65d62b3f 100644 --- a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts @@ -23,8 +23,6 @@ export class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnal this.retrieveVersionsFromField('Depends', 'r'); this.retrieveVersionsFromField('Imports', 'package'); - console.log(analyzer); - console.log(pluginConfig); return Promise.resolve(undefined); } From cfff4649cef1db300db74848eeb77e2a197be0a3 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Tue, 22 Jul 2025 16:04:02 +0200 Subject: [PATCH 17/70] lint-fix(plugin): fix mixed spaces and tabs --- src/project/plugins/flowr-analyzer-plugin.ts | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/project/plugins/flowr-analyzer-plugin.ts b/src/project/plugins/flowr-analyzer-plugin.ts index 26a4796eaaf..d9c8438b774 100644 --- a/src/project/plugins/flowr-analyzer-plugin.ts +++ b/src/project/plugins/flowr-analyzer-plugin.ts @@ -6,27 +6,27 @@ import type { PathLike } from 'fs'; export type PluginType = 'package-versions' | 'loading-order' | 'scoping' | 'file'; export interface FlowrAnalyzerPluginInterface { - readonly name: string; - readonly description: string; - readonly version: SemVer; - readonly type: PluginType; - dependencies: FlowrAnalyzerPlugin[]; + readonly name: string; + readonly description: string; + readonly version: SemVer; + readonly type: PluginType; + dependencies: FlowrAnalyzerPlugin[]; - processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise; + processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise; } export abstract class FlowrAnalyzerPlugin implements FlowrAnalyzerPluginInterface { public abstract readonly name: string; public abstract readonly description: string; public abstract readonly version: SemVer; - public abstract readonly type: PluginType; - public abstract dependencies: FlowrAnalyzerPlugin[]; - public rootPath: PathLike | undefined; + public abstract readonly type: PluginType; + public abstract dependencies: FlowrAnalyzerPlugin[]; + public rootPath: PathLike | undefined; - public setRootPath(rootPath: PathLike | undefined): void { - this.rootPath = rootPath; - } + public setRootPath(rootPath: PathLike | undefined): void { + this.rootPath = rootPath; + } - public abstract processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise; + public abstract processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise; } From 0c867b77b006a63b63f1ae7211c637a2381c6040 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 24 Jul 2025 09:29:45 +0200 Subject: [PATCH 18/70] lint-fix(plugin): unify use of `:` for type annotations --- src/project/flowr-analyzer-builder.ts | 4 ++-- .../plugins/file-plugins/flowr-analyzer-file-plugin.ts | 2 +- ...flowr-analyzer-package-versions-description-file-plugin.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 870ce41ae0e..096844f7a40 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -12,12 +12,12 @@ export class FlowrAnalyzerBuilder { private readonly request: RParseRequests; private plugins: FlowrAnalyzerPlugin[]; - public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions) : this { + public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions): this { this.flowrConfig = amendConfig(this.flowrConfig, func); return this; } - public setEngine(engine : EngineConfig['type']) { + public setEngine(engine: EngineConfig['type']) { this.flowrConfig.defaultEngine = engine; return this; } diff --git a/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts index 1b0afb4e159..5a5eea24347 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts @@ -5,7 +5,7 @@ export abstract class FlowrAnalyzerFilePlugin extends FlowrAnalyzerPlugin { public readonly type = 'file'; protected files: PathLike[] = []; - public addFiles(...files: PathLike[]) : void { + public addFiles(...files: PathLike[]): void { this.files.push(...files); } } \ No newline at end of file diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts index 3ec65d62b3f..6a7888e6451 100644 --- a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts @@ -26,7 +26,7 @@ export class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnal return Promise.resolve(undefined); } - private retrieveVersionsFromField(field: string, type?: PackageType) : void{ + private retrieveVersionsFromField(field: string, type?: PackageType): void{ for(const entry of this.descriptionFile?.get(field) || []) { const match = RegExp(/^([a-zA-Z0-9.]+)(?:\s*\(([><=~!]+)\s*([\d.]+)\))?$/).exec(entry); From 2bdc456581cad114dff3a0ff7b64b737784c3610 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 31 Jul 2025 09:44:04 +0200 Subject: [PATCH 19/70] refactor(plugin/files): move dcf-parsing to file --- .../flowr-analyzer-description-file-plugin.ts | 49 +++--------------- src/util/files.ts | 50 ++++++++++++++++++- 2 files changed, 57 insertions(+), 42 deletions(-) diff --git a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts index f1d387924f9..82e0d4218ce 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts @@ -3,57 +3,24 @@ import { SemVer } from 'semver'; import type { FlowrAnalyzer } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; -import { readLineByLine } from '../../../util/files'; -import type { PathLike } from 'fs'; +import { parseDCF } from '../../../util/files'; export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin { public readonly name = 'flowr-analyzer-description-file-plugin'; public readonly description = 'This plugin does...'; public readonly version = new SemVer('0.1.0'); public readonly dependencies: FlowrAnalyzerPlugin[] = []; - public readonly information: Map = new Map(); + public information: Map = new Map(); public async processor(_analyzer: FlowrAnalyzer, _pluginConfig: FlowrConfigOptions): Promise { - for(const file of this.files) { - const parsedDescriptionFile = await this.parseDescription(file); - for(const [key, values] of parsedDescriptionFile) { - this.information.set(key, values); - } + if(this.files.length === 0) { + throw new Error('FlowrAnalyzerDescriptionFilePlugin: No DESCRIPTION file found.'); } - } - - private async parseDescription(file: PathLike): Promise> { - const result = new Map(); - let currentKey = ''; - let currentValue = ''; - - // eslint-disable-next-line @typescript-eslint/require-await - await readLineByLine(file.toString(), async(lineBuf) => { - const line = lineBuf.toString(); - - if(/^\s/.test(line)) { - currentValue += '\n' + line.trim(); - } else { - if(currentKey) { - const values = currentValue - ? currentValue.split(/[\n,]+/).map(s => s.trim().replace(/'/g, '')).filter(s => s.length > 0) - : []; - result.set(currentKey, values); - } - - const [key, ...rest] = line.split(':'); - currentKey = key.trim(); - currentValue = rest.join(':').trim(); - } - }); - - if(currentKey) { - const values = currentValue - ? currentValue.split(/[\n,]+/).map(s => s.trim().replace(/'/g, '')).filter(s => s.length > 0) - : []; - result.set(currentKey, values); + if(this.files.length > 1){ + throw new Error('FlowrAnalyzerDescriptionFilePlugin: Found more than one DESCRIPTION file.'); } + this.information = parseDCF(this.files[0]); - return result; + return Promise.resolve(); } } \ No newline at end of file diff --git a/src/util/files.ts b/src/util/files.ts index 93c2145d74e..f5c71cd5c3c 100644 --- a/src/util/files.ts +++ b/src/util/files.ts @@ -1,4 +1,4 @@ -import fs, { promises as fsPromise } from 'fs'; +import fs, { type PathLike, promises as fsPromise } from 'fs'; import path from 'path'; import { log } from './log'; import LineByLine from 'n-readlines'; @@ -141,3 +141,51 @@ export function getParentDirectory(directory: string): string{ // apparently this is somehow the best way to do it in node, what return directory.split(path.sep).slice(0, -1).join(path.sep); } + +/** + * Parses the given file in the 'Debian Control Format'. + * @param file - The file to parse + * @returns Map containing the keys and values of the provided file. + */ +export function parseDCF(file: PathLike): Map { + const result = new Map(); + let currentKey = ''; + let currentValue = ''; + const indentRegex = new RegExp(/^\s/); + const firstColonRegex = new RegExp(/:(.*)/s); + + const fileContent = fs.readFileSync(file, 'utf-8').split('\n'); + + for(const line of fileContent) { + if(indentRegex.test(line)) { + currentValue += '\n' + line.trim(); + } else { + if(currentKey) { + const values = currentValue ? cleanValues(currentValue) : []; + result.set(currentKey, values); + } + + const [key, rest] = line.split(firstColonRegex).map(s => s.trim()); + currentKey = key.trim(); + currentValue = rest.trim(); + } + } + + if(currentKey) { + const values = currentValue ? cleanValues(currentValue) : []; + result.set(currentKey, values); + } + + console.log(result); + + return result; +} + +function cleanValues(values: string): string[] { + const splitRegex = new RegExp(/[\n,]+/); + const quotesRegex = new RegExp(/'/g); + return values + .split(splitRegex) + .map(s => s.trim().replace(quotesRegex, '')) + .filter(s => s.length > 0); +} From 3df529986e1ab8a8611cec495319764c1bab2724 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 31 Jul 2025 09:45:04 +0200 Subject: [PATCH 20/70] feat(plugin): derive package version from multiple constraints --- ...ackage-versions-description-file-plugin.ts | 4 ++- .../package-version-plugins/package.ts | 35 +++++++++++++------ test/functionality/project/package.test.ts | 19 ++++++++++ 3 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 test/functionality/project/package.test.ts diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts index 6a7888e6451..7384677009e 100644 --- a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts @@ -37,7 +37,9 @@ export class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnal const range = Package.parsePackageVersionRange(operator, version); - this.packages.push(new Package(name, range, type)); + if(range){ + this.packages.push(new Package(name, type, undefined, range)); + } } } } diff --git a/src/project/plugins/package-version-plugins/package.ts b/src/project/plugins/package-version-plugins/package.ts index 8de31f28273..5f76139cdfe 100644 --- a/src/project/plugins/package-version-plugins/package.ts +++ b/src/project/plugins/package-version-plugins/package.ts @@ -3,34 +3,49 @@ import { Range } from 'semver'; export type PackageType = 'package' | 'system' | 'r'; export class Package { - public name?: string; - public version?: Range; - public type?: PackageType; - public dependencies?: Package[]; + public name?: string; + public derivedVersion?: Range; + public type?: PackageType; + public dependencies?: Package[]; + public versionConstraints: Range[] = []; - constructor(name?: string, version?: Range, type?: PackageType, dependencies?: Package[]) { - this.addInfo(name, version, type, dependencies); + constructor(name?: string, type?: PackageType, dependencies?: Package[], ...versionConstraints: Range[]) { + this.addInfo(name, type, dependencies, ...(versionConstraints ?? [])); } - public addInfo(name?: string, versionConstraint?: Range, type?: PackageType, dependencies?: Package[]): void { + public addInfo(name?: string, type?: PackageType, dependencies?: Package[], ...versionConstraints: Range[]): void { if(name !== undefined) { this.name = name; } - if(versionConstraint !== undefined) { - this.version = versionConstraint; - } if(type !== undefined) { this.type = type; } if(dependencies !== undefined) { this.dependencies = dependencies; } + if(versionConstraints !== undefined) { + this.derivedVersion ??= versionConstraints[0]; + + for(const constraint of versionConstraints) { + if(!this.derivedVersion?.intersects(constraint)) { + throw Error('Version constraint mismatch!'); + } + this.versionConstraints.push(constraint); + this.derivedVersion = this.deriveVersion(); + } + } } public getInfo(): this { return this; } + public deriveVersion(): Range | undefined { + return this.versionConstraints.length > 0 + ? new Range(this.versionConstraints.map(c => c.raw).join(' ')) + : undefined; + } + public static parsePackageVersionRange(constraint?: string, version?: string): Range | undefined { if(version) { return constraint ? new Range(constraint + version) : new Range(version); diff --git a/test/functionality/project/package.test.ts b/test/functionality/project/package.test.ts new file mode 100644 index 00000000000..c989e24ca61 --- /dev/null +++ b/test/functionality/project/package.test.ts @@ -0,0 +1,19 @@ +import { describe, test , assert } from 'vitest'; +import { Package } from '../../../src/project/plugins/package-version-plugins/package'; +import { Range } from 'semver'; + +describe('DESCRIPTION-file', function() { + describe.sequential('Parsing', function() { + test('Library-Versions-Plugin', () => { + const p1 = new Package(); + p1.addInfo('Test Package', 'package', undefined, new Range('>= 1.3')); + p1.addInfo(undefined, undefined, undefined, new Range('<= 2.3')); + p1.addInfo(undefined, undefined, undefined, new Range('>= 1.5')); + p1.addInfo(undefined, undefined, undefined, new Range('<= 2.2.5')); + + console.log(p1); + + assert.isTrue(p1.derivedVersion?.test('1.7.0')); + }); + }); +}); \ No newline at end of file From cb935b11ac50712b938eca3d4e53a191b3e0758b Mon Sep 17 00:00:00 2001 From: stimjannik Date: Wed, 6 Aug 2025 16:37:07 +0200 Subject: [PATCH 21/70] wip(plugin): add libraries to dependency query --- src/project/flowr-project.ts | 42 +++++++++++++++++++ src/queries/base-query-format.ts | 4 ++ .../dependencies-query-executor.ts | 39 +++++++++++------ .../dependencies-query-format.ts | 5 ++- .../dataflow/query/dependencies-query.test.ts | 6 ++- test/functionality/project/package.test.ts | 5 ++- 6 files changed, 83 insertions(+), 18 deletions(-) create mode 100644 src/project/flowr-project.ts diff --git a/src/project/flowr-project.ts b/src/project/flowr-project.ts new file mode 100644 index 00000000000..6c62ae62e43 --- /dev/null +++ b/src/project/flowr-project.ts @@ -0,0 +1,42 @@ +import type { FlowrAnalyzer } from './flowr-analyzer'; +import type { FlowrAnalyzerBuilder } from './flowr-analyzer-builder'; +import type { FlowrAnalyzerPlugin } from './plugins/flowr-analyzer-plugin'; +import type { PathLike } from 'fs'; +import path from 'path'; +import type { Package } from './plugins/package-version-plugins/package'; +import { FlowrAnalyzerDescriptionFilePlugin } from './plugins/file-plugins/flowr-analyzer-description-file-plugin'; +import { + FlowrAnalyzerPackageVersionsDescriptionFilePlugin +} from './plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin'; +import type { FlowrConfigOptions } from '../config'; + +export interface FlowrProject { + analyzer: FlowrAnalyzer; + builder: FlowrAnalyzerBuilder; + plugins: FlowrAnalyzerPlugin[]; + libraries: Package[]; + projectRoot: PathLike; +} + +export async function getDummyFlowrProject(){ + const exampleFlowrProject : FlowrProject = { + analyzer: {} as FlowrAnalyzer, + builder: {} as FlowrAnalyzerBuilder, + plugins: [], + projectRoot: path.resolve('test/testfiles/project/'), + libraries: [] + }; + + const descriptionFilePlugin = new FlowrAnalyzerDescriptionFilePlugin(); + descriptionFilePlugin.addFiles(path.resolve('test/testfiles/project/DESCRIPTION')); + + const flowrAnalyzerPackageVersionsDescriptionFilePlugin = new FlowrAnalyzerPackageVersionsDescriptionFilePlugin(); + flowrAnalyzerPackageVersionsDescriptionFilePlugin.dependencies = [descriptionFilePlugin]; + + await flowrAnalyzerPackageVersionsDescriptionFilePlugin + .processor({} as FlowrAnalyzer, {} as FlowrConfigOptions) + + exampleFlowrProject.libraries = flowrAnalyzerPackageVersionsDescriptionFilePlugin.packages; + + return exampleFlowrProject; +} \ No newline at end of file diff --git a/src/queries/base-query-format.ts b/src/queries/base-query-format.ts index 5a9809692e7..ba3cdfc621c 100644 --- a/src/queries/base-query-format.ts +++ b/src/queries/base-query-format.ts @@ -1,6 +1,8 @@ import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { DataflowInformation } from '../dataflow/info'; import type { FlowrConfigOptions } from '../config'; +import type { LibraryInfo } from './catalog/dependencies-query/dependencies-query-format'; +import type { FlowrProject } from '../project/flowr-project'; export interface BaseQueryFormat { /** used to select the query type :) */ @@ -16,6 +18,8 @@ export interface BaseQueryResult { } export interface BasicQueryData { + project?: FlowrProject; + readonly library?: LibraryInfo; readonly ast: NormalizedAst; readonly dataflow: DataflowInformation; readonly config: FlowrConfigOptions; diff --git a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts index d7fc04b6907..c2326a0154f 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts @@ -26,6 +26,8 @@ import type { DependencyInfoLinkAttachedInfo, FunctionInfo } from './function-in import { DependencyInfoLinkConstraint } from './function-info/function-info'; import { CallTargets } from '../call-context-query/identify-link-to-last-call-relation'; import { getArgumentStringValue } from '../../../dataflow/eval/resolve/resolve-argument'; +import type { Package } from '../../../project/plugins/package-version-plugins/package'; +import { getDummyFlowrProject } from '../../../project/flowr-project'; function collectNamespaceAccesses(data: BasicQueryData, libraries: LibraryInfo[]) { /* for libraries, we have to additionally track all uses of `::` and `:::`, for this we currently simply traverse all uses */ @@ -33,9 +35,11 @@ function collectNamespaceAccesses(data: BasicQueryData, libraries: LibraryInfo[] if(n.type === RType.Symbol && n.namespace) { /* we should improve the identification of ':::' */ libraries.push({ - nodeId: n.info.id, - functionName: (n.info.fullLexeme ?? n.lexeme).includes(':::') ? ':::' : '::', - libraryName: n.namespace, + nodeId: n.info.id, + functionName: (n.info.fullLexeme ?? n.lexeme).includes(':::') ? ':::' : '::', + libraryName: n.namespace, + versionConstraints: data.library?.versionConstraints ?? undefined, + derivedVersion: data.library?.derivedVersion ?? undefined }); } }); @@ -47,6 +51,8 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly } const now = Date.now(); + // data.project = getDummyFlowrProject(); + const [query] = queries; const ignoreDefault = query.ignoreDefaultFunctions ?? false; const libraryFunctions = getFunctionsToCheck(query.libraryFunctions, ignoreDefault, LibraryFunctions); @@ -74,12 +80,14 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly return get?.info.fullLexeme ?? get?.lexeme; } - const libraries: LibraryInfo[] = getResults(data, results, 'library', libraryFunctions, (id, vertex, argId, value, linkedIds) => ({ - nodeId: id, - functionName: vertex.name, - lexemeOfArgument: getLexeme(value, argId), - libraryName: value ?? Unknown, - linkedIds: linkedIds?.length ? linkedIds : undefined + const libraries: LibraryInfo[] = getResults(data, results, 'library', libraryFunctions, (id, vertex, argId, value, linkedIds, library?: Package) => ({ + nodeId: id, + functionName: vertex.name, + lexemeOfArgument: getLexeme(value, argId), + libraryName: value ?? Unknown, + linkedIds: linkedIds?.length ? linkedIds : undefined, + versionConstraints: library?.versionConstraints ?? undefined, + derivedVersion: library?.derivedVersion ?? undefined })); if(!ignoreDefault) { @@ -108,7 +116,6 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly lexemeOfArgument: getLexeme(value, argId), linkedIds: linkedIds?.length? linkedIds : undefined })); - return { '.meta': { timing: Date.now() - now @@ -135,7 +142,8 @@ type MakeDependencyInfo = ( vertex: DataflowGraphVertexFunctionCall, argumentId: NodeId | undefined, argumentValue: string | undefined, - linkedIds: undefined | readonly NodeId[] + linkedIds: undefined | readonly NodeId[], + library?: Package, ) => T | undefined; function dropInfoOnLinkedIds(linkedIds: readonly (NodeId | { id: NodeId, info: object })[] | undefined): NodeId[] | undefined{ @@ -148,6 +156,7 @@ function dropInfoOnLinkedIds(linkedIds: readonly (NodeId | { id: NodeId, info: o function getResults(data: BasicQueryData, results: CallContextQueryResult, kind: string, functions: FunctionInfo[], makeInfo: MakeDependencyInfo): T[] { const kindEntries = Object.entries(results?.kinds[kind]?.subkinds ?? {}); return kindEntries.flatMap(([name, results]) => results.flatMap(({ id, linkedIds }) => { + const vertex = data.dataflow.graph.getVertex(id) as DataflowGraphVertexFunctionCall; const info = functions.find(f => f.name === name) as FunctionInfo; @@ -163,9 +172,15 @@ function getResults(data: BasicQueryData, results: Cal return record ? [record as T] : []; } const results: T[] = []; + for(const [arg, values] of foundValues.entries()) { for(const value of values) { - const result = compactRecord(makeInfo(id, vertex, arg, value, dropInfoOnLinkedIds(linkedIds))); + let pkg = undefined; + if(name === 'library'){ + console.log(data.project?.libraries); + pkg = data.project?.libraries.find(p => p.name === value); + } + const result = compactRecord(makeInfo(id, vertex, arg, value, dropInfoOnLinkedIds(linkedIds), pkg)); if(result) { results.push(result as T); } diff --git a/src/queries/catalog/dependencies-query/dependencies-query-format.ts b/src/queries/catalog/dependencies-query/dependencies-query-format.ts index d6f4695cc86..a857c9024cc 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-format.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-format.ts @@ -6,6 +6,7 @@ import { printAsMs } from '../../../util/text/time'; import Joi from 'joi'; import { executeDependenciesQuery } from './dependencies-query-executor'; import type { FunctionInfo } from './function-info/function-info'; +import type { Range } from 'semver'; export const Unknown = 'unknown'; @@ -32,7 +33,7 @@ export interface DependencyInfo extends Record{ /** the lexeme is presented whenever the specific info is of {@link Unknown} */ lexemeOfArgument?: string; } -export type LibraryInfo = (DependencyInfo & { libraryName: 'unknown' | string }) +export type LibraryInfo = (DependencyInfo & { libraryName: 'unknown' | string, versionConstraints?: Range[], derivedVersion?: Range}) export type SourceInfo = (DependencyInfo & { file: string }) export type ReadInfo = (DependencyInfo & { source: string }) export type WriteInfo = (DependencyInfo & { destination: 'stdout' | string }) @@ -69,7 +70,7 @@ export const DependenciesQueryDefinition = { asciiSummarizer: (formatter, _processed, queryResults, result) => { const out = queryResults as QueryResults<'dependencies'>['dependencies']; result.push(`Query: ${bold('dependencies', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); - printResultSection('Libraries', out.libraries, result, l => `\`${l.libraryName}\``); + printResultSection('Libraries', out.libraries, result, l => `\`${l.libraryName}\`, Version: \`${l.derivedVersion?.raw ?? 'unknown'}\``); printResultSection('Sourced Files', out.sourcedFiles, result, s => `\`${s.file}\``); printResultSection('Read Data', out.readData, result, r => `\`${r.source}\``); printResultSection('Written Data', out.writtenData, result, w => `\`${w.destination}\``); diff --git a/test/functionality/dataflow/query/dependencies-query.test.ts b/test/functionality/dataflow/query/dependencies-query.test.ts index 8e0e0b98e62..afadb3c1499 100644 --- a/test/functionality/dataflow/query/dependencies-query.test.ts +++ b/test/functionality/dataflow/query/dependencies-query.test.ts @@ -11,6 +11,10 @@ import type { SingleSlicingCriterion } from '../../../../src/slicing/criterion/p import { describe } from 'vitest'; import { withTreeSitter } from '../../_helper/shell'; + + + + const emptyDependencies: Omit = { libraries: [], sourcedFiles: [], readData: [], writtenData: [] }; function decodeIds(res: Partial, idMap: AstIdMap): Partial { @@ -77,7 +81,6 @@ describe('Dependencies Query', withTreeSitter(parser => { { nodeId: '1@require', functionName: 'require', libraryName: 'unknown', lexemeOfArgument: 'c' } ] }); - testQuery('Library with variable', 'a <- "ggplot2"\nb <- TRUE\nlibrary(a,character.only=b)', { libraries: [ { nodeId: '3@library', functionName: 'library', libraryName: 'ggplot2' } ] }); @@ -184,7 +187,6 @@ describe('Dependencies Query', withTreeSitter(parser => { { nodeId: '2@library', functionName: 'library', libraryName: 'g' } ] }); - describe('Custom', () => { const readCustomFile: Partial = { libraryFunctions: [{ package: 'custom', name: 'custom.library', argIdx: 1, argName: 'file' }] diff --git a/test/functionality/project/package.test.ts b/test/functionality/project/package.test.ts index c989e24ca61..b28f39c04a9 100644 --- a/test/functionality/project/package.test.ts +++ b/test/functionality/project/package.test.ts @@ -1,6 +1,6 @@ import { describe, test , assert } from 'vitest'; import { Package } from '../../../src/project/plugins/package-version-plugins/package'; -import { Range } from 'semver'; +import { minVersion, Range } from 'semver'; describe('DESCRIPTION-file', function() { describe.sequential('Parsing', function() { @@ -11,7 +11,8 @@ describe('DESCRIPTION-file', function() { p1.addInfo(undefined, undefined, undefined, new Range('>= 1.5')); p1.addInfo(undefined, undefined, undefined, new Range('<= 2.2.5')); - console.log(p1); + console.log(p1.derivedVersion); + console.log(minVersion(p1.derivedVersion as Range)); assert.isTrue(p1.derivedVersion?.test('1.7.0')); }); From 96982a1df3a94704d9110bfbf694acfb104d9743 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 7 Aug 2025 10:48:49 +0200 Subject: [PATCH 22/70] lint-fix(plugin): add missing semicolon --- src/project/flowr-project.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/project/flowr-project.ts b/src/project/flowr-project.ts index 6c62ae62e43..b06e655ad81 100644 --- a/src/project/flowr-project.ts +++ b/src/project/flowr-project.ts @@ -34,7 +34,7 @@ export async function getDummyFlowrProject(){ flowrAnalyzerPackageVersionsDescriptionFilePlugin.dependencies = [descriptionFilePlugin]; await flowrAnalyzerPackageVersionsDescriptionFilePlugin - .processor({} as FlowrAnalyzer, {} as FlowrConfigOptions) + .processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); exampleFlowrProject.libraries = flowrAnalyzerPackageVersionsDescriptionFilePlugin.packages; From dcb24ee7576aa3bc649c91de7014e8e61e60a211 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 7 Aug 2025 10:49:52 +0200 Subject: [PATCH 23/70] feat(plugin): add description file infos to dependency query --- src/cli/repl/commands/repl-query.ts | 5 ++++- src/queries/base-query-format.ts | 12 +++++------ .../dependencies-query-executor.ts | 21 ++++++------------- 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index 07a3a6d0afb..673efaac027 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -12,6 +12,7 @@ import { jsonReplacer } from '../../../util/json'; import { asciiSummaryOfQueryResult } from '../../../queries/query-print'; import type { KnownParser } from '../../../r-bridge/parser'; import type { FlowrConfigOptions } from '../../../config'; +import { getDummyFlowrProject } from '../../../project/flowr-project'; async function getDataflow(config: FlowrConfigOptions, parser: KnownParser, remainingLine: string) { @@ -66,9 +67,11 @@ async function processQueryArgs(line: string, parser: KnownParser, output: ReplO parsedQuery = [{ type: 'call-context', callName: query }]; } + const dummyProject = await getDummyFlowrProject(); + const processed = await getDataflow(config, parser, args.join(' ')); return { - query: executeQueries({ dataflow: processed.dataflow, ast: processed.normalize, config: config }, parsedQuery), + query: executeQueries({ dataflow: processed.dataflow, ast: processed.normalize, config: config, libraries: dummyProject.libraries }, parsedQuery), processed }; } diff --git a/src/queries/base-query-format.ts b/src/queries/base-query-format.ts index ba3cdfc621c..bc664d4983a 100644 --- a/src/queries/base-query-format.ts +++ b/src/queries/base-query-format.ts @@ -1,8 +1,7 @@ import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { DataflowInformation } from '../dataflow/info'; import type { FlowrConfigOptions } from '../config'; -import type { LibraryInfo } from './catalog/dependencies-query/dependencies-query-format'; -import type { FlowrProject } from '../project/flowr-project'; +import type { Package } from '../project/plugins/package-version-plugins/package'; export interface BaseQueryFormat { /** used to select the query type :) */ @@ -18,9 +17,8 @@ export interface BaseQueryResult { } export interface BasicQueryData { - project?: FlowrProject; - readonly library?: LibraryInfo; - readonly ast: NormalizedAst; - readonly dataflow: DataflowInformation; - readonly config: FlowrConfigOptions; + readonly libraries?: Package[]; + readonly ast: NormalizedAst; + readonly dataflow: DataflowInformation; + readonly config: FlowrConfigOptions; } diff --git a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts index c2326a0154f..df20d89596f 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts @@ -27,7 +27,6 @@ import { DependencyInfoLinkConstraint } from './function-info/function-info'; import { CallTargets } from '../call-context-query/identify-link-to-last-call-relation'; import { getArgumentStringValue } from '../../../dataflow/eval/resolve/resolve-argument'; import type { Package } from '../../../project/plugins/package-version-plugins/package'; -import { getDummyFlowrProject } from '../../../project/flowr-project'; function collectNamespaceAccesses(data: BasicQueryData, libraries: LibraryInfo[]) { /* for libraries, we have to additionally track all uses of `::` and `:::`, for this we currently simply traverse all uses */ @@ -38,8 +37,8 @@ function collectNamespaceAccesses(data: BasicQueryData, libraries: LibraryInfo[] nodeId: n.info.id, functionName: (n.info.fullLexeme ?? n.lexeme).includes(':::') ? ':::' : '::', libraryName: n.namespace, - versionConstraints: data.library?.versionConstraints ?? undefined, - derivedVersion: data.library?.derivedVersion ?? undefined + versionConstraints: data.libraries?.find(f => f.name === n.namespace)?.versionConstraints ?? [], + derivedVersion: data.libraries?.find(f => f.name === n.namespace)?.derivedVersion ?? undefined, }); } }); @@ -50,9 +49,6 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly log.warn('Dependencies query expects only up to one query, but got ', queries.length, 'only using the first query'); } const now = Date.now(); - - // data.project = getDummyFlowrProject(); - const [query] = queries; const ignoreDefault = query.ignoreDefaultFunctions ?? false; const libraryFunctions = getFunctionsToCheck(query.libraryFunctions, ignoreDefault, LibraryFunctions); @@ -80,14 +76,14 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly return get?.info.fullLexeme ?? get?.lexeme; } - const libraries: LibraryInfo[] = getResults(data, results, 'library', libraryFunctions, (id, vertex, argId, value, linkedIds, library?: Package) => ({ + const libraries: LibraryInfo[] = getResults(data, results, 'library', libraryFunctions, (id, vertex, argId, value, linkedIds) => ({ nodeId: id, functionName: vertex.name, lexemeOfArgument: getLexeme(value, argId), libraryName: value ?? Unknown, linkedIds: linkedIds?.length ? linkedIds : undefined, - versionConstraints: library?.versionConstraints ?? undefined, - derivedVersion: library?.derivedVersion ?? undefined + versionConstraints: data.libraries?.find(f => f.name === value)?.versionConstraints ?? [], + derivedVersion: data.libraries?.find(f => f.name === value)?.derivedVersion ?? undefined, })); if(!ignoreDefault) { @@ -175,12 +171,7 @@ function getResults(data: BasicQueryData, results: Cal for(const [arg, values] of foundValues.entries()) { for(const value of values) { - let pkg = undefined; - if(name === 'library'){ - console.log(data.project?.libraries); - pkg = data.project?.libraries.find(p => p.name === value); - } - const result = compactRecord(makeInfo(id, vertex, arg, value, dropInfoOnLinkedIds(linkedIds), pkg)); + const result = compactRecord(makeInfo(id, vertex, arg, value, dropInfoOnLinkedIds(linkedIds))); if(result) { results.push(result as T); } From 76fd1c5c32df5b5d24812efc018605b62a5b39d7 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 7 Aug 2025 13:56:10 +0200 Subject: [PATCH 24/70] feat-fix(plugin): namespace returns undefined for constraints if no pkg --- .../catalog/dependencies-query/dependencies-query-executor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts index df20d89596f..b5a346449e4 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts @@ -37,7 +37,7 @@ function collectNamespaceAccesses(data: BasicQueryData, libraries: LibraryInfo[] nodeId: n.info.id, functionName: (n.info.fullLexeme ?? n.lexeme).includes(':::') ? ':::' : '::', libraryName: n.namespace, - versionConstraints: data.libraries?.find(f => f.name === n.namespace)?.versionConstraints ?? [], + versionConstraints: data.libraries?.find(f => f.name === n.namespace)?.versionConstraints ?? undefined, derivedVersion: data.libraries?.find(f => f.name === n.namespace)?.derivedVersion ?? undefined, }); } @@ -82,7 +82,7 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly lexemeOfArgument: getLexeme(value, argId), libraryName: value ?? Unknown, linkedIds: linkedIds?.length ? linkedIds : undefined, - versionConstraints: data.libraries?.find(f => f.name === value)?.versionConstraints ?? [], + versionConstraints: data.libraries?.find(f => f.name === value)?.versionConstraints ?? undefined, derivedVersion: data.libraries?.find(f => f.name === value)?.derivedVersion ?? undefined, })); From 12df85007e7cb4cd40593a644e53221619a01ec2 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 7 Aug 2025 14:29:13 +0200 Subject: [PATCH 25/70] test(dependencies-query): add tests for packages with versions --- src/util/files.ts | 2 -- test/functionality/_helper/query.ts | 5 ++++- .../dataflow/query/dependencies-query.test.ts | 22 ++++++++++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/util/files.ts b/src/util/files.ts index f5c71cd5c3c..78a3c43e2d5 100644 --- a/src/util/files.ts +++ b/src/util/files.ts @@ -176,8 +176,6 @@ export function parseDCF(file: PathLike): Map { result.set(currentKey, values); } - console.log(result); - return result; } diff --git a/test/functionality/_helper/query.ts b/test/functionality/_helper/query.ts index f469690c538..9deb203acf5 100644 --- a/test/functionality/_helper/query.ts +++ b/test/functionality/_helper/query.ts @@ -16,6 +16,7 @@ import { cfgToMermaidUrl } from '../../../src/util/mermaid/cfg'; import { defaultConfigOptions } from '../../../src/config'; import type { KnownParser } from '../../../src/r-bridge/parser'; import { extractCfg } from '../../../src/control-flow/extract-cfg'; +import {getDummyFlowrProject} from "../../../src/project/flowr-project"; function normalizeResults(result: QueryResults): QueryResultsWithoutMeta { @@ -72,7 +73,9 @@ export function assertQuery< getId: deterministicCountingIdGenerator(0) }, defaultConfigOptions).allRemainingSteps(); - const result = executeQueries({ dataflow: info.dataflow, ast: info.normalize, config: defaultConfigOptions }, queries); + const dummyProject = await getDummyFlowrProject(); + + const result = executeQueries({ dataflow: info.dataflow, ast: info.normalize, config: defaultConfigOptions, libraries: dummyProject.libraries }, queries); log.info(`total query time: ${result['.meta'].timing.toFixed(0)}ms (~1ms accuracy)`); diff --git a/test/functionality/dataflow/query/dependencies-query.test.ts b/test/functionality/dataflow/query/dependencies-query.test.ts index afadb3c1499..1cfcd52d262 100644 --- a/test/functionality/dataflow/query/dependencies-query.test.ts +++ b/test/functionality/dataflow/query/dependencies-query.test.ts @@ -9,6 +9,7 @@ import type { AstIdMap } from '../../../../src/r-bridge/lang-4.x/ast/model/proce import type { SingleSlicingCriterion } from '../../../../src/slicing/criterion/parse'; import { describe } from 'vitest'; +import { Range } from 'semver'; import { withTreeSitter } from '../../_helper/shell'; @@ -82,7 +83,7 @@ describe('Dependencies Query', withTreeSitter(parser => { ] }); testQuery('Library with variable', 'a <- "ggplot2"\nb <- TRUE\nlibrary(a,character.only=b)', { libraries: [ - { nodeId: '3@library', functionName: 'library', libraryName: 'ggplot2' } + { nodeId: '3@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] } ] }); // for now, we want a better or (https://github.com/flowr-analysis/flowr/issues/1342) @@ -172,8 +173,8 @@ describe('Dependencies Query', withTreeSitter(parser => { ] }); testQuery('Using a vector by variable (real world)', 'packages <- c("ggplot2", "dplyr", "tidyr")\nlapply(packages, library, character.only = TRUE)', { libraries: [ - { nodeId: '2@library', functionName: 'library', libraryName: 'ggplot2' }, - { nodeId: '2@library', functionName: 'library', libraryName: 'dplyr' }, + { nodeId: '2@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] }, + { nodeId: '2@library', functionName: 'library', libraryName: 'dplyr', derivedVersion: new Range('>=1.4.0'), versionConstraints: [new Range('>=1.4.0')] }, { nodeId: '2@library', functionName: 'library', libraryName: 'tidyr' } ] }); @@ -187,6 +188,21 @@ describe('Dependencies Query', withTreeSitter(parser => { { nodeId: '2@library', functionName: 'library', libraryName: 'g' } ] }); + testQuery('Library with version', 'library(ggplot2)', { libraries: [ + { nodeId: '1@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] } + ] }); + + testQuery('Libraries with versions', 'library(ggplot2)\nlibrary(dplyr)', { libraries: [ + { nodeId: '1@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] }, + { nodeId: '2@library', functionName: 'library', libraryName: 'dplyr', derivedVersion: new Range('>=1.4.0'), versionConstraints: [new Range('>=1.4.0')] }, + ] }); + + testQuery('Libraries with and without versions', 'library(ggplot2)\nlibrary(dplyr)\nlibrary(tidyr)', { libraries: [ + { nodeId: '1@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] }, + { nodeId: '2@library', functionName: 'library', libraryName: 'dplyr', derivedVersion: new Range('>=1.4.0'), versionConstraints: [new Range('>=1.4.0')] }, + { nodeId: '3@library', functionName: 'library', libraryName: 'tidyr', derivedVersion: undefined, versionConstraints: undefined }, + ] }); + describe('Custom', () => { const readCustomFile: Partial = { libraryFunctions: [{ package: 'custom', name: 'custom.library', argIdx: 1, argName: 'file' }] From 6ae00e728ade63ed1e7634594d826f1566ad5f58 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Thu, 7 Aug 2025 14:30:13 +0200 Subject: [PATCH 26/70] lint-fix(dependencies-query): fix braces --- test/functionality/_helper/query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionality/_helper/query.ts b/test/functionality/_helper/query.ts index 9deb203acf5..9eb6626f98a 100644 --- a/test/functionality/_helper/query.ts +++ b/test/functionality/_helper/query.ts @@ -16,7 +16,7 @@ import { cfgToMermaidUrl } from '../../../src/util/mermaid/cfg'; import { defaultConfigOptions } from '../../../src/config'; import type { KnownParser } from '../../../src/r-bridge/parser'; import { extractCfg } from '../../../src/control-flow/extract-cfg'; -import {getDummyFlowrProject} from "../../../src/project/flowr-project"; +import { getDummyFlowrProject } from '../../../src/project/flowr-project'; function normalizeResults(result: QueryResults): QueryResultsWithoutMeta { From e26fe2dcd909549f8306f5c1d9cfe37e821b7291 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 15 Aug 2025 16:51:27 +0200 Subject: [PATCH 27/70] feat(builder): add cfg and force param --- src/project/flowr-analyzer.ts | 56 ++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index b8da8ebcd42..f21c2a5d936 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -1,22 +1,25 @@ import type { FlowrConfigOptions } from '../config'; import type { RParseRequests } from '../r-bridge/retriever'; import { requestFromInput } from '../r-bridge/retriever'; -import { createDataflowPipeline, createSlicePipeline } from '../core/steps/pipeline/default-pipelines'; +import { createDataflowPipeline, createNormalizePipeline } from '../core/steps/pipeline/default-pipelines'; import { FlowrAnalyzerBuilder } from './flowr-analyzer-builder'; import { graphToMermaidUrl } from '../util/mermaid/dfg'; import type { KnownParser } from '../r-bridge/parser'; -import type { SlicingCriteria } from '../slicing/criterion/parse'; import type { Queries, SupportedQueryTypes } from '../queries/query'; import { executeQueries } from '../queries/query'; import type { DataflowInformation } from '../dataflow/info'; import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; +import { extractCfg } from '../control-flow/extract-cfg'; +import type { ControlFlowInformation } from '../control-flow/control-flow-graph'; export class FlowrAnalyzer { private readonly flowrConfig: FlowrConfigOptions; private readonly request: RParseRequests; private readonly parser: KnownParser; + private ast = undefined as unknown as NormalizedAst; private dataflowInfo = undefined as unknown as DataflowInformation; + private controlflowInfo = undefined as unknown as ControlFlowInformation; constructor(config: FlowrConfigOptions, parser: KnownParser, request: RParseRequests) { this.flowrConfig = config; @@ -24,29 +27,50 @@ export class FlowrAnalyzer { this.parser = parser; } - public async dataflow() { - const result = await createDataflowPipeline( + public async normalizedAst(force?: boolean): Promise { + if(this.ast && !force) { + return this.ast; + } + + const result = await createNormalizePipeline( this.parser, { request: this.request }, this.flowrConfig).allRemainingSteps(); - this.dataflowInfo = result.dataflow; this.ast = result.normalize; - return result; + return result.normalize; } - public async slice(criterion: SlicingCriteria) { - return createSlicePipeline( + public async dataflow(force?: boolean): Promise { + if(this.dataflowInfo && !force) { + return this.dataflowInfo; + } + + const result = await createDataflowPipeline( this.parser, - { - request: this.request, - criterion: criterion - }, + { request: this.request }, this.flowrConfig).allRemainingSteps(); + this.dataflowInfo = result.dataflow; + this.ast = result.normalize; + return result.dataflow; + } + + public async controlflow(force?: boolean): Promise { + if(this.controlflowInfo && !force) { + return this.controlflowInfo; + } + + if(force || !this.ast) { + await this.normalizedAst(force); + } + + const result = extractCfg(this.ast, this.flowrConfig); + this.controlflowInfo = result; + return result; } - public async query(query: Queries) { + public async query(query: Queries, force?: boolean) { if(!this.dataflowInfo) { - await this.dataflow(); + await this.dataflow(force); } return executeQueries({ ast: this.ast, dataflow: this.dataflowInfo, config: this.flowrConfig }, query); } @@ -67,9 +91,7 @@ async function main() { subkind: 'test-subkind', callName: /foo/ }]); - const slice = await analyzer.slice(['3@foo']); - console.log(graphToMermaidUrl(result.dataflow.graph)); - console.log(slice.reconstruct); + console.log(graphToMermaidUrl(result.graph)); console.log(query); } From f8f1462e83570d19e4298d3adae305b8519e2dd8 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 15 Aug 2025 17:05:40 +0200 Subject: [PATCH 28/70] feat(builder): add reset function --- src/project/flowr-analyzer.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index f21c2a5d936..70a8bdd6620 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -27,6 +27,12 @@ export class FlowrAnalyzer { this.parser = parser; } + public reset() { + this.ast = undefined as unknown as NormalizedAst; + this.dataflowInfo = undefined as unknown as DataflowInformation; + this.controlflowInfo = undefined as unknown as ControlFlowInformation; + } + public async normalizedAst(force?: boolean): Promise { if(this.ast && !force) { return this.ast; From 57a57fa4b8a257c2d94bc7bafd0efe90cd391932 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Tue, 19 Aug 2025 13:36:17 +0200 Subject: [PATCH 29/70] feat(analyzer): use analyzer in repl --- src/cli/repl/commands/repl-cfg.ts | 56 +++++++++------------- src/cli/repl/commands/repl-commands.ts | 10 ++-- src/cli/repl/commands/repl-dataflow.ts | 64 ++++++++++--------------- src/cli/repl/commands/repl-execute.ts | 1 + src/cli/repl/commands/repl-lineage.ts | 1 + src/cli/repl/commands/repl-main.ts | 24 +++++++++- src/cli/repl/commands/repl-normalize.ts | 39 ++++++--------- src/cli/repl/commands/repl-parse.ts | 3 +- src/cli/repl/commands/repl-query.ts | 2 + src/cli/repl/commands/repl-quit.ts | 1 + src/cli/repl/commands/repl-version.ts | 1 + src/cli/repl/core.ts | 24 ++++++++-- src/documentation/doc-util/doc-repl.ts | 4 +- src/project/flowr-analyzer-builder.ts | 14 +++++- src/project/flowr-analyzer.ts | 39 +++------------ 15 files changed, 142 insertions(+), 141 deletions(-) diff --git a/src/cli/repl/commands/repl-cfg.ts b/src/cli/repl/commands/repl-cfg.ts index cca2e9843dc..288a2445ac9 100644 --- a/src/cli/repl/commands/repl-cfg.ts +++ b/src/cli/repl/commands/repl-cfg.ts @@ -1,35 +1,21 @@ -import type { ReplCommand, ReplOutput } from './repl-main'; -import { extractCfg } from '../../../control-flow/extract-cfg'; -import { createDataflowPipeline } from '../../../core/steps/pipeline/default-pipelines'; -import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever'; +import type { ReplCodeCommand, ReplOutput } from './repl-main'; +import { fileProtocol } from '../../../r-bridge/retriever'; import { cfgToMermaid, cfgToMermaidUrl } from '../../../util/mermaid/cfg'; -import type { KnownParser } from '../../../r-bridge/parser'; import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi'; import type { ControlFlowInformation } from '../../../control-flow/control-flow-graph'; import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { CfgSimplificationPassName } from '../../../control-flow/cfg-simplification'; import { DefaultCfgSimplificationOrder } from '../../../control-flow/cfg-simplification'; -import type { FlowrConfigOptions } from '../../../config'; - -async function controlflow(parser: KnownParser, remainingLine: string, config: FlowrConfigOptions) { - return await createDataflowPipeline(parser, { - request: requestFromInput(remainingLine.trim()) - }, config).allRemainingSteps(); -} - -function handleString(code: string): string { - return code.startsWith('"') ? JSON.parse(code) as string : code; -} +import type { FlowrAnalyzer } from '../../../project/flowr-analyzer'; function formatInfo(out: ReplOutput, type: string): string { return out.formatter.format(`Copied ${type} to clipboard.`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); } -async function produceAndPrintCfg(shell: KnownParser, remainingLine: string, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string, config: FlowrConfigOptions) { - const result = await controlflow(shell, handleString(remainingLine), config); - - const cfg = extractCfg(result.normalize, config, result.dataflow.graph, [...DefaultCfgSimplificationOrder, ...simplifications]); - const mermaid = cfgConverter(cfg, result.normalize); +async function produceAndPrintCfg(analyzer: FlowrAnalyzer, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string) { + const cfg = await analyzer.controlflow([...DefaultCfgSimplificationOrder, ...simplifications]); + const result = await analyzer.normalizedAst(); + const mermaid = cfgConverter(cfg, result); output.stdout(mermaid); try { const clipboard = await import('clipboardy'); @@ -39,45 +25,49 @@ async function produceAndPrintCfg(shell: KnownParser, remainingLine: string, out } } -export const controlflowCommand: ReplCommand = { +export const controlflowCommand: ReplCodeCommand = { description: `Get mermaid code for the control-flow graph of R code, start with '${fileProtocol}' to indicate a file`, + usesAnalyzer: true, usageExample: ':controlflow', aliases: [ 'cfg', 'cf' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - await produceAndPrintCfg(parser, remainingLine, output, [], cfgToMermaid, config); + fn: async({ output, analyzer }) => { + await produceAndPrintCfg(analyzer, output, [], cfgToMermaid); } }; -export const controlflowStarCommand: ReplCommand = { +export const controlflowStarCommand: ReplCodeCommand = { description: 'Returns the URL to mermaid.live', + usesAnalyzer: true, usageExample: ':controlflow*', aliases: [ 'cfg*', 'cf*' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - await produceAndPrintCfg(parser, remainingLine, output, [], cfgToMermaidUrl, config); + fn: async({ output, analyzer }) => { + await produceAndPrintCfg(analyzer, output, [], cfgToMermaidUrl); } }; -export const controlflowBbCommand: ReplCommand = { +export const controlflowBbCommand: ReplCodeCommand = { description: `Get mermaid code for the control-flow graph with basic blocks, start with '${fileProtocol}' to indicate a file`, + usesAnalyzer: true, usageExample: ':controlflowbb', aliases: [ 'cfgb', 'cfb' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - await produceAndPrintCfg(parser, remainingLine, output, ['to-basic-blocks'], cfgToMermaid, config); + fn: async({ output, analyzer }) => { + await produceAndPrintCfg(analyzer, output, ['to-basic-blocks'], cfgToMermaid); } }; -export const controlflowBbStarCommand: ReplCommand = { +export const controlflowBbStarCommand: ReplCodeCommand = { description: 'Returns the URL to mermaid.live', + usesAnalyzer: true, usageExample: ':controlflowbb*', aliases: [ 'cfgb*', 'cfb*' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - await produceAndPrintCfg(parser, remainingLine, output, ['to-basic-blocks' ], cfgToMermaidUrl, config); + fn: async({ output, analyzer }) => { + await produceAndPrintCfg(analyzer, output, ['to-basic-blocks' ], cfgToMermaidUrl); } }; diff --git a/src/cli/repl/commands/repl-commands.ts b/src/cli/repl/commands/repl-commands.ts index 1ae05ea2ef0..615064ef6f6 100644 --- a/src/cli/repl/commands/repl-commands.ts +++ b/src/cli/repl/commands/repl-commands.ts @@ -1,6 +1,6 @@ import { quitCommand } from './repl-quit'; import { stdioCaptureProcessor, waitOnScript } from '../execute'; -import type { ReplCommand } from './repl-main'; +import type { ReplBaseCommand, ReplCodeCommand, ReplCommand } from './repl-main'; import { rawPrompt } from '../prompt'; import { versionCommand } from './repl-version'; import { parseCommand } from './repl-parse'; @@ -21,7 +21,7 @@ import { scripts } from '../../common/scripts-info'; import { lineageCommand } from './repl-lineage'; import { queryCommand, queryStarCommand } from './repl-query'; -function printHelpForScript(script: [string, ReplCommand], f: OutputFormatter, starredVersion?: ReplCommand): string { +function printHelpForScript(script: [string, ReplBaseCommand], f: OutputFormatter, starredVersion?: ReplBaseCommand): string { let base = ` ${bold(padCmd(':' + script[0] + (starredVersion ? '[*]' : '')), f)}${script[1].description}`; if(starredVersion) { base += ` (star: ${starredVersion.description})`; @@ -49,6 +49,7 @@ function printCommandHelp(formatter: OutputFormatter) { export const helpCommand: ReplCommand = { description: 'Show help information', + usesAnalyzer: false, script: false, usageExample: ':help', aliases: [ 'h', '?' ], @@ -79,7 +80,7 @@ You can combine commands by separating them with a semicolon ${bold(';',output.f /** * All commands that should be available in the REPL. */ -const _commands: Record = { +const _commands: Record = { 'help': helpCommand, 'quit': quitCommand, 'version': versionCommand, @@ -122,6 +123,7 @@ export function getReplCommands() { aliases: [], script: true, usageExample: `:${script} --help`, + usesAnalyzer: false, fn: async({ output, remainingLine }) => { // check if the target *module* exists in the current directory, else try two dirs up, otherwise, fail with a message let path = `${__dirname}/${target}`; @@ -177,7 +179,7 @@ function initCommandMapping() { * Get the command for a given command name or alias. * @param command - The name of the command (without the leading `:`) */ -export function getCommand(command: string): ReplCommand | undefined { +export function getCommand(command: string): ReplCodeCommand | ReplCommand | undefined { if(commandMapping === undefined) { initCommandMapping(); } diff --git a/src/cli/repl/commands/repl-dataflow.ts b/src/cli/repl/commands/repl-dataflow.ts index 8017771d887..ec209ce5da4 100644 --- a/src/cli/repl/commands/repl-dataflow.ts +++ b/src/cli/repl/commands/repl-dataflow.ts @@ -1,93 +1,81 @@ -import type { ReplCommand, ReplOutput } from './repl-main'; -import { createDataflowPipeline } from '../../../core/steps/pipeline/default-pipelines'; -import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever'; +import type { ReplCodeCommand, ReplOutput } from './repl-main'; +import { fileProtocol } from '../../../r-bridge/retriever'; import { graphToMermaid, graphToMermaidUrl } from '../../../util/mermaid/dfg'; -import type { KnownParser } from '../../../r-bridge/parser'; import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi'; -import type { FlowrConfigOptions } from '../../../config'; - -/** - * Obtain the dataflow graph using a known parser (such as the {@link RShell} or {@link TreeSitterExecutor}). - */ -async function replGetDataflow(config: FlowrConfigOptions, parser: KnownParser, code: string) { - return await createDataflowPipeline(parser, { - request: requestFromInput(code.trim()) - }, config).allRemainingSteps(); -} - -function handleString(code: string): string { - return code.startsWith('"') ? JSON.parse(code) as string : code; -} function formatInfo(out: ReplOutput, type: string, timing: number): string { return out.formatter.format(`Copied ${type} to clipboard (dataflow: ${timing}ms).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); } -export const dataflowCommand: ReplCommand = { +export const dataflowCommand: ReplCodeCommand = { description: `Get mermaid code for the dataflow graph, start with '${fileProtocol}' to indicate a file`, + usesAnalyzer: true, usageExample: ':dataflow', aliases: [ 'd', 'df' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - const result = await replGetDataflow(config, parser, handleString(remainingLine)); - const mermaid = graphToMermaid({ graph: result.dataflow.graph, includeEnvironments: false }).string; + fn: async({ output, analyzer }) => { + const result = await analyzer.dataflow(); + const mermaid = graphToMermaid({ graph: result.graph, includeEnvironments: false }).string; output.stdout(mermaid); try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid code', result.dataflow['.meta'].timing)); + output.stdout(formatInfo(output, 'mermaid code', 0)); // TODO } catch{ /* do nothing this is a service thing */ } } }; -export const dataflowStarCommand: ReplCommand = { +export const dataflowStarCommand: ReplCodeCommand = { description: 'Returns the URL to mermaid.live', + usesAnalyzer: true, usageExample: ':dataflow*', aliases: [ 'd*', 'df*' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - const result = await replGetDataflow(config, parser, handleString(remainingLine)); - const mermaid = graphToMermaidUrl(result.dataflow.graph, false); + fn: async({ output, analyzer }) => { + const result = await analyzer.dataflow(); + const mermaid = graphToMermaidUrl(result.graph, false); output.stdout(mermaid); try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid url', result.dataflow['.meta'].timing)); + output.stdout(formatInfo(output, 'mermaid url', 0)); // TODO } catch{ /* do nothing this is a service thing */ } } }; -export const dataflowSimplifiedCommand: ReplCommand = { +export const dataflowSimplifiedCommand: ReplCodeCommand = { description: `Get mermaid code for the simplified dataflow graph, start with '${fileProtocol}' to indicate a file`, + usesAnalyzer: true, usageExample: ':dataflowsimple', aliases: [ 'ds', 'dfs' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - const result = await replGetDataflow(config, parser, handleString(remainingLine)); - const mermaid = graphToMermaid({ graph: result.dataflow.graph, includeEnvironments: false, simplified: true }).string; + fn: async({ output, analyzer }) => { + const result = await analyzer.dataflow(); + const mermaid = graphToMermaid({ graph: result.graph, includeEnvironments: false, simplified: true }).string; output.stdout(mermaid); try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid code', result.dataflow['.meta'].timing)); + output.stdout(formatInfo(output, 'mermaid code', 0)); // TODO } catch{ /* do nothing this is a service thing */ } } }; -export const dataflowSimpleStarCommand: ReplCommand = { +export const dataflowSimpleStarCommand: ReplCodeCommand = { description: 'Returns the URL to mermaid.live', + usesAnalyzer: true, usageExample: ':dataflowsimple*', aliases: [ 'ds*', 'dfs*' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - const result = await replGetDataflow(config, parser, handleString(remainingLine)); - const mermaid = graphToMermaidUrl(result.dataflow.graph, false, undefined, true); + fn: async({ output, analyzer }) => { + const result = await analyzer.dataflow(); + const mermaid = graphToMermaidUrl(result.graph, false, undefined, true); output.stdout(mermaid); try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid url', result.dataflow['.meta'].timing)); + output.stdout(formatInfo(output, 'mermaid url', 0)); // TODO } catch{ /* do nothing this is a service thing */ } } }; diff --git a/src/cli/repl/commands/repl-execute.ts b/src/cli/repl/commands/repl-execute.ts index 8253e18b7ac..32e80b8e897 100644 --- a/src/cli/repl/commands/repl-execute.ts +++ b/src/cli/repl/commands/repl-execute.ts @@ -27,6 +27,7 @@ async function executeRShellCommand(output: ReplOutput, shell: RShell, statement export const executeCommand: ReplCommand = { description: 'Execute the given code as R code (essentially similar to using now command). This requires the `--r-session-access` flag to be set and requires the r-shell engine.', + usesAnalyzer: false, usageExample: ':execute', aliases: [ 'e', 'r' ], script: false, diff --git a/src/cli/repl/commands/repl-lineage.ts b/src/cli/repl/commands/repl-lineage.ts index c3610b555fc..d6960edf100 100644 --- a/src/cli/repl/commands/repl-lineage.ts +++ b/src/cli/repl/commands/repl-lineage.ts @@ -67,6 +67,7 @@ export function getLineage(criterion: SingleSlicingCriterion, graph: DataflowGra export const lineageCommand: ReplCommand = { description: 'Get the lineage of an R object', + usesAnalyzer: false, usageExample: ':lineage', aliases: ['lin'], script: false, diff --git a/src/cli/repl/commands/repl-main.ts b/src/cli/repl/commands/repl-main.ts index 5eeb8dad31d..c84c9784901 100644 --- a/src/cli/repl/commands/repl-main.ts +++ b/src/cli/repl/commands/repl-main.ts @@ -2,6 +2,7 @@ import type { OutputFormatter } from '../../../util/text/ansi'; import { formatter } from '../../../util/text/ansi'; import type { KnownParser } from '../../../r-bridge/parser'; import type { FlowrConfigOptions } from '../../../config'; +import type { FlowrAnalyzer } from '../../../project/flowr-analyzer'; /** * Defines the main interface for output of the repl. @@ -30,19 +31,25 @@ export const standardReplOutput: ReplOutput = { /** * Information passed to each repl command function */ + export interface ReplCommandInformation { output: ReplOutput, + allowRSessionAccess: boolean, parser: KnownParser, remainingLine: string, - allowRSessionAccess: boolean, config: FlowrConfigOptions } +export interface ReplCodeCommandInformation { + output: ReplOutput, + analyzer: FlowrAnalyzer +} + /** * Content of a single command in the repl. * The command may execute an external script or simply call *flowR* functions. */ -export interface ReplCommand { +export interface ReplBaseCommand { /** Aliases of the command (without the leading colon), every alias must be unique (this is checked at runtime) */ aliases: string[] /** A human-readable description of what the command does */ @@ -51,9 +58,22 @@ export interface ReplCommand { script: boolean /** Example of how to use the command, for example `:slicer --help` */ usageExample: string +} + +export interface ReplCommand extends ReplBaseCommand { + usesAnalyzer: false; /** * Function to execute when the command is invoked, it must not write to the command line but instead use the output handler. * Furthermore, it has to obey the formatter defined in the {@link ReplOutput}. */ fn: (info: ReplCommandInformation) => Promise | void } + +export interface ReplCodeCommand extends ReplBaseCommand { + usesAnalyzer: true; + /** + * Function to execute when the command is invoked, it must not write to the command line but instead use the output handler. + * Furthermore, it has to obey the formatter defined in the {@link ReplOutput}. + */ + fn: (info: ReplCodeCommandInformation) => Promise | void +} diff --git a/src/cli/repl/commands/repl-normalize.ts b/src/cli/repl/commands/repl-normalize.ts index 94af01b5b8a..bfd0809f3f2 100644 --- a/src/cli/repl/commands/repl-normalize.ts +++ b/src/cli/repl/commands/repl-normalize.ts @@ -1,55 +1,44 @@ -import type { ReplCommand, ReplOutput } from './repl-main'; -import { createNormalizePipeline } from '../../../core/steps/pipeline/default-pipelines'; -import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever'; +import type { ReplCodeCommand, ReplOutput } from './repl-main'; +import { fileProtocol } from '../../../r-bridge/retriever'; import { normalizedAstToMermaid, normalizedAstToMermaidUrl } from '../../../util/mermaid/ast'; -import type { KnownParser } from '../../../r-bridge/parser'; import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi'; -import type { FlowrConfigOptions } from '../../../config'; - -async function normalize(parser: KnownParser, remainingLine: string, config: FlowrConfigOptions) { - return await createNormalizePipeline(parser, { - request: requestFromInput(remainingLine.trim()) - }, config).allRemainingSteps(); -} - -function handleString(code: string): string { - return code.startsWith('"') ? JSON.parse(code) as string : code; -} function formatInfo(out: ReplOutput, type: string, timing: number): string { return out.formatter.format(`Copied ${type} to clipboard (normalize: ${timing}ms).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); } -export const normalizeCommand: ReplCommand = { +export const normalizeCommand: ReplCodeCommand = { description: `Get mermaid code for the normalized AST of R code, start with '${fileProtocol}' to indicate a file`, + usesAnalyzer: true, usageExample: ':normalize', aliases: [ 'n' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - const result = await normalize(parser, handleString(remainingLine), config); - const mermaid = normalizedAstToMermaid(result.normalize.ast); + fn: async({ output, analyzer }) => { + const result = await analyzer.normalizedAst(); + const mermaid = normalizedAstToMermaid(result.ast); output.stdout(mermaid); try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid url', result.normalize['.meta'].timing)); + output.stdout(formatInfo(output, 'mermaid url', 0)); // TODO } catch{ /* do nothing this is a service thing */ } } }; -export const normalizeStarCommand: ReplCommand = { +export const normalizeStarCommand: ReplCodeCommand = { description: 'Returns the URL to mermaid.live', + usesAnalyzer: true, usageExample: ':normalize*', aliases: [ 'n*' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - const result = await normalize(parser, handleString(remainingLine), config); - const mermaid = normalizedAstToMermaidUrl(result.normalize.ast); + fn: async({ output, analyzer }) => { + const result = await analyzer.normalizedAst(); + const mermaid = normalizedAstToMermaidUrl(result.ast); output.stdout(mermaid); try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid url', result.normalize['.meta'].timing)); + output.stdout(formatInfo(output, 'mermaid url', 0)); // TODO } catch{ /* do nothing this is a service thing */ } } }; diff --git a/src/cli/repl/commands/repl-parse.ts b/src/cli/repl/commands/repl-parse.ts index 6e47531f686..06df3d8a531 100644 --- a/src/cli/repl/commands/repl-parse.ts +++ b/src/cli/repl/commands/repl-parse.ts @@ -153,9 +153,10 @@ function depthListToTextTree(list: Readonly, f: OutputFormatter): str return result; } - +// TODO TSchoeller Should we use the analyzer here? export const parseCommand: ReplCommand = { description: `Prints ASCII Art of the parsed, unmodified AST, start with '${fileProtocol}' to indicate a file`, + usesAnalyzer: false, usageExample: ':parse', aliases: [ 'p' ], script: false, diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index 673efaac027..f207fff74be 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -78,6 +78,7 @@ async function processQueryArgs(line: string, parser: KnownParser, output: ReplO export const queryCommand: ReplCommand = { description: `Query the given R code, start with '${fileProtocol}' to indicate a file. The query is to be a valid query in json format (use 'help' to get more information).`, + usesAnalyzer: false, usageExample: ':query "" ', aliases: [], script: false, @@ -93,6 +94,7 @@ export const queryCommand: ReplCommand = { export const queryStarCommand: ReplCommand = { description: 'Similar to query, but returns the output in json format.', + usesAnalyzer: false, usageExample: ':query* ', aliases: [ ], script: false, diff --git a/src/cli/repl/commands/repl-quit.ts b/src/cli/repl/commands/repl-quit.ts index aff414a3390..ef8947e16ef 100644 --- a/src/cli/repl/commands/repl-quit.ts +++ b/src/cli/repl/commands/repl-quit.ts @@ -3,6 +3,7 @@ import { log } from '../../../util/log'; export const quitCommand: ReplCommand = { description: 'End the repl', + usesAnalyzer: false, aliases: [ 'q', 'exit' ], usageExample: ':quit', script: false, diff --git a/src/cli/repl/commands/repl-version.ts b/src/cli/repl/commands/repl-version.ts index 7f24989ba91..cc9deeb9965 100644 --- a/src/cli/repl/commands/repl-version.ts +++ b/src/cli/repl/commands/repl-version.ts @@ -37,6 +37,7 @@ export async function printVersionInformation(output: ReplOutput, parser: KnownP export const versionCommand: ReplCommand = { description: 'Prints the version of flowR as well as the current version of R', + usesAnalyzer: false, aliases: [], usageExample: ':version', script: false, diff --git a/src/cli/repl/core.ts b/src/cli/repl/core.ts index f2b98c23531..82de7ee35a8 100644 --- a/src/cli/repl/core.ts +++ b/src/cli/repl/core.ts @@ -13,7 +13,7 @@ import { splitAtEscapeSensitive } from '../../util/text/args'; import { FontStyles } from '../../util/text/ansi'; import { getCommand, getCommandNames } from './commands/repl-commands'; import { getValidOptionsForCompletion, scripts } from '../common/scripts-info'; -import { fileProtocol } from '../../r-bridge/retriever'; +import { fileProtocol, requestFromInput } from '../../r-bridge/retriever'; import type { ReplOutput } from './commands/repl-main'; import { standardReplOutput } from './commands/repl-main'; import { RShell, RShellReviveOptions } from '../../r-bridge/shell'; @@ -22,6 +22,7 @@ import type { KnownParser } from '../../r-bridge/parser'; import { log, LogLevel } from '../../util/log'; import type { FlowrConfigOptions } from '../../config'; import { getEngineConfig } from '../../config'; +import { FlowrAnalyzerBuilder } from '../../project/flowr-analyzer-builder'; let _replCompleterKeywords: string[] | undefined = undefined; function replCompleterKeywords() { @@ -80,6 +81,10 @@ export function makeDefaultReplReadline(): readline.ReadLineOptions { }; } +function handleString(code: string): string { + return code.startsWith('"') ? JSON.parse(code) as string : code; +} + async function replProcessStatement(output: ReplOutput, statement: string, parser: KnownParser, allowRSessionAccess: boolean, config: FlowrConfigOptions): Promise { if(statement.startsWith(':')) { const command = statement.slice(1).split(' ')[0].toLowerCase(); @@ -87,7 +92,20 @@ async function replProcessStatement(output: ReplOutput, statement: string, parse const bold = (s: string) => output.formatter.format(s, { style: FontStyles.Bold }); if(processor) { try { - await processor.fn({ output, parser, remainingLine: statement.slice(command.length + 2).trim(), allowRSessionAccess, config }); + const remainingLine = statement.slice(command.length + 2).trim(); + if(processor.usesAnalyzer) { + const request = requestFromInput(handleString(remainingLine)); + // TODO TSchoeller engine/parser + // TODO TSchoeller Ideally the analyzer would also be used for query commands + // TODO TSchoeller Is this the right level to create the analyzer instance? + const analyzer = await new FlowrAnalyzerBuilder(request) + .setConfig(config) + .setParser(parser) + .build(); + await processor.fn({ output, analyzer }); + } else { + await processor.fn({ output, parser, remainingLine, allowRSessionAccess, config }); + } } catch(e){ output.stdout(`${bold(`Failed to execute command ${command}`)}: ${(e as Error)?.message}. Using the ${bold('--verbose')} flag on startup may provide additional information.\n`); if(log.settings.minLevel < LogLevel.Fatal) { @@ -154,7 +172,7 @@ export interface FlowrReplOptions extends MergeableRecord { * */ export async function repl( - config : FlowrConfigOptions, + config: FlowrConfigOptions, { parser = new RShell(getEngineConfig(config, 'r-shell'), { revive: RShellReviveOptions.Always }), rl = readline.createInterface(makeDefaultReplReadline()), diff --git a/src/documentation/doc-util/doc-repl.ts b/src/documentation/doc-util/doc-repl.ts index f203202196f..f2217b9e879 100644 --- a/src/documentation/doc-util/doc-repl.ts +++ b/src/documentation/doc-util/doc-repl.ts @@ -1,4 +1,4 @@ -import type { ReplCommand, ReplOutput } from '../../cli/repl/commands/repl-main'; +import type { ReplBaseCommand, ReplOutput } from '../../cli/repl/commands/repl-main'; import { getReplCommands } from '../../cli/repl/commands/repl-commands'; import { getReplCommand } from './doc-cli-option'; import { textWithTooltip } from '../../util/html-hover-over'; @@ -11,7 +11,7 @@ import { versionReplString } from '../../cli/repl/print-version'; import type { KnownParser } from '../../r-bridge/parser'; import { defaultConfigOptions } from '../../config'; -function printHelpForScript(script: [string, ReplCommand], starredVersion?: ReplCommand): string { +function printHelpForScript(script: [string, ReplBaseCommand], starredVersion?: ReplBaseCommand): string { let base = `| **${getReplCommand(script[0], false, starredVersion !== undefined)}** | ${script[1].description}`; if(starredVersion) { base += ` (star: ${starredVersion.description})`; diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 096844f7a40..d96bb315758 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -9,6 +9,7 @@ import type { FlowrAnalyzerPlugin } from './plugins/flowr-analyzer-plugin'; export class FlowrAnalyzerBuilder { private flowrConfig: DeepWritable = cloneConfig(defaultConfigOptions); + private parser?: KnownParser; private readonly request: RParseRequests; private plugins: FlowrAnalyzerPlugin[]; @@ -17,6 +18,16 @@ export class FlowrAnalyzerBuilder { return this; } + public setConfig(config: FlowrConfigOptions) { + this.flowrConfig = config; + return this; + } + + public setParser(parser: KnownParser) { + this.parser = parser; + return this; + } + public setEngine(engine: EngineConfig['type']) { this.flowrConfig.defaultEngine = engine; return this; @@ -39,7 +50,8 @@ export class FlowrAnalyzerBuilder { public async build(): Promise { const engines = await retrieveEngineInstances(this.flowrConfig); - const parser = engines.engines[engines.default] as KnownParser; + // TODO TSchoeller Is this complexity necessary? + const parser = this.parser ?? engines.engines[engines.default] as KnownParser; return new FlowrAnalyzer( this.flowrConfig, diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 70a8bdd6620..19c1e028bc6 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -1,21 +1,19 @@ import type { FlowrConfigOptions } from '../config'; import type { RParseRequests } from '../r-bridge/retriever'; -import { requestFromInput } from '../r-bridge/retriever'; import { createDataflowPipeline, createNormalizePipeline } from '../core/steps/pipeline/default-pipelines'; -import { FlowrAnalyzerBuilder } from './flowr-analyzer-builder'; -import { graphToMermaidUrl } from '../util/mermaid/dfg'; import type { KnownParser } from '../r-bridge/parser'; import type { Queries, SupportedQueryTypes } from '../queries/query'; import { executeQueries } from '../queries/query'; -import type { DataflowInformation } from '../dataflow/info'; -import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; import { extractCfg } from '../control-flow/extract-cfg'; import type { ControlFlowInformation } from '../control-flow/control-flow-graph'; +import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { DataflowInformation } from '../dataflow/info'; +import type { CfgSimplificationPassName } from '../control-flow/cfg-simplification'; export class FlowrAnalyzer { - private readonly flowrConfig: FlowrConfigOptions; - private readonly request: RParseRequests; - private readonly parser: KnownParser; + public readonly flowrConfig: FlowrConfigOptions; + private readonly request: RParseRequests; + private readonly parser: KnownParser; private ast = undefined as unknown as NormalizedAst; private dataflowInfo = undefined as unknown as DataflowInformation; @@ -60,7 +58,7 @@ export class FlowrAnalyzer { return result.dataflow; } - public async controlflow(force?: boolean): Promise { + public async controlflow(simplifications?: readonly CfgSimplificationPassName[], force?: boolean): Promise { if(this.controlflowInfo && !force) { return this.controlflowInfo; } @@ -80,27 +78,4 @@ export class FlowrAnalyzer { } return executeQueries({ ast: this.ast, dataflow: this.dataflowInfo, config: this.flowrConfig }, query); } -} - -async function main() { - const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('x <- 1\nfoo <- function(){}\nfoo()')) - .setEngine('r-shell') - .amendConfig(c => { - c.solver.pointerTracking = true; - return c; - }) - .build(); - const result = await analyzer.dataflow(); - const query = await analyzer.query([{ - type: 'call-context', - kind: 'test-kind', - subkind: 'test-subkind', - callName: /foo/ - }]); - console.log(graphToMermaidUrl(result.graph)); - console.log(query); -} - -if(require.main === module) { - void main(); } \ No newline at end of file From 6a6c498e7a748b60ca75726ee5bf5b62df59e192 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Tue, 19 Aug 2025 13:49:31 +0200 Subject: [PATCH 30/70] lint-fix: type-annotation spacing --- src/cli/flowr.ts | 2 +- src/core/pipeline-executor.ts | 2 +- src/dataflow/eval/resolve/alias-tracking.ts | 2 +- src/dataflow/eval/resolve/resolve-argument.ts | 4 ++-- src/project/flowr-project.ts | 2 +- .../parser/main/internal/structure/normalize-expressions.ts | 2 +- .../control-flow/dead-code/cfg-dead-code.test.ts | 2 +- .../slicing/configuration/source-finding.test.ts | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cli/flowr.ts b/src/cli/flowr.ts index 19ead3036e4..08c774bb040 100644 --- a/src/cli/flowr.ts +++ b/src/cli/flowr.ts @@ -87,7 +87,7 @@ if(options['no-ansi']) { setFormatter(voidFormatter); } -function createConfig() : FlowrConfigOptions { +function createConfig(): FlowrConfigOptions { let config: FlowrConfigOptions | undefined; if(options['config-json']) { diff --git a/src/core/pipeline-executor.ts b/src/core/pipeline-executor.ts index d16404b7ae3..52b84304d07 100644 --- a/src/core/pipeline-executor.ts +++ b/src/core/pipeline-executor.ts @@ -163,7 +163,7 @@ export class PipelineExecutor

{ } - public getResults(intermediate?:false): PipelineOutput

+ public getResults(intermediate?: false): PipelineOutput

public getResults(intermediate: true): Partial> public getResults(intermediate: boolean): PipelineOutput

/** diff --git a/src/dataflow/eval/resolve/alias-tracking.ts b/src/dataflow/eval/resolve/alias-tracking.ts index 734a5762695..baef8c84871 100644 --- a/src/dataflow/eval/resolve/alias-tracking.ts +++ b/src/dataflow/eval/resolve/alias-tracking.ts @@ -147,7 +147,7 @@ export function getAliases(sourceIds: readonly NodeId[], dataflow: DataflowGraph * @param full - Whether to track aliases on resolve * @param resolve - Variable resolve mode */ -export function resolveIdToValue(id: NodeId | RNodeWithParent | undefined, { environment, graph, idMap, full = true, resolve } : ResolveInfo): ResolveResult { +export function resolveIdToValue(id: NodeId | RNodeWithParent | undefined, { environment, graph, idMap, full = true, resolve }: ResolveInfo): ResolveResult { if(id === undefined) { return Top; } diff --git a/src/dataflow/eval/resolve/resolve-argument.ts b/src/dataflow/eval/resolve/resolve-argument.ts index 02c62b32fd7..7966044e2ed 100644 --- a/src/dataflow/eval/resolve/resolve-argument.ts +++ b/src/dataflow/eval/resolve/resolve-argument.ts @@ -25,7 +25,7 @@ export function getArgumentStringValue( vertex: DataflowGraphVertexFunctionCall, argumentIndex: number | 'unnamed' | undefined, argumentName: string | undefined, - resolveValue : boolean | 'library' | undefined + resolveValue: boolean | 'library' | undefined ): Map> | undefined { if(argumentName) { const arg = vertex?.args.findIndex(arg => arg !== EmptyArgument && arg.name === argumentName); @@ -91,7 +91,7 @@ function hasCharacterOnly(variableResolve: VariableResolve, graph: DataflowGraph } } -function resolveBasedOnConfig(variableResolve: VariableResolve, graph: DataflowGraph, vertex: DataflowGraphVertexFunctionCall, argument: RNodeWithParent, environment: REnvironmentInformation | undefined, idMap: Map | undefined, resolveValue : boolean | 'library' | undefined): string[] | undefined { +function resolveBasedOnConfig(variableResolve: VariableResolve, graph: DataflowGraph, vertex: DataflowGraphVertexFunctionCall, argument: RNodeWithParent, environment: REnvironmentInformation | undefined, idMap: Map | undefined, resolveValue: boolean | 'library' | undefined): string[] | undefined { let full = true; if(!resolveValue) { full = false; diff --git a/src/project/flowr-project.ts b/src/project/flowr-project.ts index b06e655ad81..9b037a1383c 100644 --- a/src/project/flowr-project.ts +++ b/src/project/flowr-project.ts @@ -19,7 +19,7 @@ export interface FlowrProject { } export async function getDummyFlowrProject(){ - const exampleFlowrProject : FlowrProject = { + const exampleFlowrProject: FlowrProject = { analyzer: {} as FlowrAnalyzer, builder: {} as FlowrAnalyzerBuilder, plugins: [], diff --git a/src/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-expressions.ts b/src/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-expressions.ts index 7e128b48e78..8ac69a36be9 100644 --- a/src/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-expressions.ts +++ b/src/r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-expressions.ts @@ -129,7 +129,7 @@ function handleExpressionList(raw: readonly NamedJsonEntry[]): HandledExpression } -function processBraces([start, end]: [start: NamedJsonEntry, end: NamedJsonEntry], processed: readonly RNode[], comments: RComment[], data: NormalizerData) : RExpressionList { +function processBraces([start, end]: [start: NamedJsonEntry, end: NamedJsonEntry], processed: readonly RNode[], comments: RComment[], data: NormalizerData): RExpressionList { const [newStart, newEnd] = [tryNormalizeSymbol(data, [start]), tryNormalizeSymbol(data, [end])]; guard(newStart !== undefined && newEnd !== undefined, () => `expected both start and end to be symbols, but ${JSON.stringify(start, jsonReplacer)} :: ${JSON.stringify(end, jsonReplacer)}`); return { diff --git a/test/functionality/control-flow/dead-code/cfg-dead-code.test.ts b/test/functionality/control-flow/dead-code/cfg-dead-code.test.ts index 19cd3d37394..29081b1eb71 100644 --- a/test/functionality/control-flow/dead-code/cfg-dead-code.test.ts +++ b/test/functionality/control-flow/dead-code/cfg-dead-code.test.ts @@ -12,7 +12,7 @@ interface CfgDeadCodeArgs { } describe('Control Flow Graph', withTreeSitter(parser => { - function assertDeadCode(code: string, { reachableFromStart, unreachableFromStart } : CfgDeadCodeArgs): void { + function assertDeadCode(code: string, { reachableFromStart, unreachableFromStart }: CfgDeadCodeArgs): void { assertCfg(parser, code, { graph: new ControlFlowGraph() }, { diff --git a/test/functionality/slicing/configuration/source-finding.test.ts b/test/functionality/slicing/configuration/source-finding.test.ts index beb035b62dd..11533534b4e 100644 --- a/test/functionality/slicing/configuration/source-finding.test.ts +++ b/test/functionality/slicing/configuration/source-finding.test.ts @@ -27,7 +27,7 @@ describe('source finding', () => { function assertSourceFound(path: string, shouldBe: string[], referenceChain: readonly RParseRequest[] = []) { test(`finds source for ${path}`, () => { - const resolveSource : FlowrLaxSourcingOptions = { + const resolveSource: FlowrLaxSourcingOptions = { dropPaths: DropPathsOption.All, ignoreCapitalization: true, inferWorkingDirectory: InferWorkingDirectory.ActiveScript, From 9066eb51a4af5101933d24bab14cc9c8d414591b Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Thu, 21 Aug 2025 21:25:57 +0200 Subject: [PATCH 31/70] feat-fix: re-introduce timing info to repl --- src/cli/repl/commands/repl-dataflow.ts | 14 ++++++++------ src/cli/repl/commands/repl-normalize.ts | 9 +++++---- src/cli/repl/core.ts | 1 - src/core/pipeline-executor.ts | 2 +- src/core/steps/pipeline/pipeline.ts | 10 ++++++++++ src/project/flowr-analyzer.ts | 19 +++++++++++++++---- .../static-slice-query-executor.ts | 6 +++--- 7 files changed, 42 insertions(+), 19 deletions(-) diff --git a/src/cli/repl/commands/repl-dataflow.ts b/src/cli/repl/commands/repl-dataflow.ts index ec209ce5da4..ca5b05e1282 100644 --- a/src/cli/repl/commands/repl-dataflow.ts +++ b/src/cli/repl/commands/repl-dataflow.ts @@ -2,9 +2,11 @@ import type { ReplCodeCommand, ReplOutput } from './repl-main'; import { fileProtocol } from '../../../r-bridge/retriever'; import { graphToMermaid, graphToMermaidUrl } from '../../../util/mermaid/dfg'; import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi'; +import type { PipelinePerStepMetaInformation } from '../../../core/steps/pipeline/pipeline'; -function formatInfo(out: ReplOutput, type: string, timing: number): string { - return out.formatter.format(`Copied ${type} to clipboard (dataflow: ${timing}ms).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); +function formatInfo(out: ReplOutput, type: string, meta: PipelinePerStepMetaInformation ): string { + return out.formatter.format(`Copied ${type} to clipboard (dataflow: ${meta['.meta'].cached ? 'cached' : meta['.meta'].timing + 'ms'}).`, + { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); } export const dataflowCommand: ReplCodeCommand = { @@ -20,7 +22,7 @@ export const dataflowCommand: ReplCodeCommand = { try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid code', 0)); // TODO + output.stdout(formatInfo(output, 'mermaid code', result)); } catch{ /* do nothing this is a service thing */ } } }; @@ -38,7 +40,7 @@ export const dataflowStarCommand: ReplCodeCommand = { try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid url', 0)); // TODO + output.stdout(formatInfo(output, 'mermaid url', result)); } catch{ /* do nothing this is a service thing */ } } }; @@ -57,7 +59,7 @@ export const dataflowSimplifiedCommand: ReplCodeCommand = { try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid code', 0)); // TODO + output.stdout(formatInfo(output, 'mermaid code', result)); } catch{ /* do nothing this is a service thing */ } } }; @@ -75,7 +77,7 @@ export const dataflowSimpleStarCommand: ReplCodeCommand = { try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid url', 0)); // TODO + output.stdout(formatInfo(output, 'mermaid url', result)); } catch{ /* do nothing this is a service thing */ } } }; diff --git a/src/cli/repl/commands/repl-normalize.ts b/src/cli/repl/commands/repl-normalize.ts index bfd0809f3f2..ae7e5b3b9cd 100644 --- a/src/cli/repl/commands/repl-normalize.ts +++ b/src/cli/repl/commands/repl-normalize.ts @@ -2,9 +2,10 @@ import type { ReplCodeCommand, ReplOutput } from './repl-main'; import { fileProtocol } from '../../../r-bridge/retriever'; import { normalizedAstToMermaid, normalizedAstToMermaidUrl } from '../../../util/mermaid/ast'; import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi'; +import type { PipelinePerStepMetaInformation } from '../../../core/steps/pipeline/pipeline'; -function formatInfo(out: ReplOutput, type: string, timing: number): string { - return out.formatter.format(`Copied ${type} to clipboard (normalize: ${timing}ms).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); +function formatInfo(out: ReplOutput, type: string, meta: PipelinePerStepMetaInformation): string { + return out.formatter.format(`Copied ${type} to clipboard (normalize: ${meta['.meta'].cached ? 'cached' : meta['.meta'].timing + 'ms'}).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); } export const normalizeCommand: ReplCodeCommand = { @@ -20,7 +21,7 @@ export const normalizeCommand: ReplCodeCommand = { try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid url', 0)); // TODO + output.stdout(formatInfo(output, 'mermaid url', result)); } catch{ /* do nothing this is a service thing */ } } }; @@ -38,7 +39,7 @@ export const normalizeStarCommand: ReplCodeCommand = { try { const clipboard = await import('clipboardy'); clipboard.default.writeSync(mermaid); - output.stdout(formatInfo(output, 'mermaid url', 0)); // TODO + output.stdout(formatInfo(output, 'mermaid url', result)); } catch{ /* do nothing this is a service thing */ } } }; diff --git a/src/cli/repl/core.ts b/src/cli/repl/core.ts index 82de7ee35a8..41f00d5b606 100644 --- a/src/cli/repl/core.ts +++ b/src/cli/repl/core.ts @@ -95,7 +95,6 @@ async function replProcessStatement(output: ReplOutput, statement: string, parse const remainingLine = statement.slice(command.length + 2).trim(); if(processor.usesAnalyzer) { const request = requestFromInput(handleString(remainingLine)); - // TODO TSchoeller engine/parser // TODO TSchoeller Ideally the analyzer would also be used for query commands // TODO TSchoeller Is this the right level to create the analyzer instance? const analyzer = await new FlowrAnalyzerBuilder(request) diff --git a/src/core/pipeline-executor.ts b/src/core/pipeline-executor.ts index 52b84304d07..03b70da1617 100644 --- a/src/core/pipeline-executor.ts +++ b/src/core/pipeline-executor.ts @@ -206,7 +206,7 @@ export class PipelineExecutor

{ const [step, result] = this._doNextStep(expectedStepName); const awaitedResult = await result; - this.output[step as PipelineStepNames

] = { ...awaitedResult, '.meta': { timing: Date.now() - start } }; + this.output[step as PipelineStepNames

] = { ...awaitedResult, '.meta': { timing: Date.now() - start, cached: false } }; this.stepCounter++; return { name: step as PassedName, result: awaitedResult }; diff --git a/src/core/steps/pipeline/pipeline.ts b/src/core/steps/pipeline/pipeline.ts index af940ba8ab5..2705cc3a2e7 100644 --- a/src/core/steps/pipeline/pipeline.ts +++ b/src/core/steps/pipeline/pipeline.ts @@ -47,6 +47,16 @@ export interface PipelinePerStepMetaInformation { * the required accuracy is dependent on the measuring system, but usually at around 1 ms. */ readonly timing: number + /** + * The result was newly created + */ + readonly cached: false; + } | { + /** + * The result was cached. + * Thus, no timing information is available. + */ + readonly cached: true; } } diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 19c1e028bc6..89c4eeda4c0 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -9,6 +9,7 @@ import type { ControlFlowInformation } from '../control-flow/control-flow-graph' import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { DataflowInformation } from '../dataflow/info'; import type { CfgSimplificationPassName } from '../control-flow/cfg-simplification'; +import type { PipelinePerStepMetaInformation } from '../core/steps/pipeline/pipeline'; export class FlowrAnalyzer { public readonly flowrConfig: FlowrConfigOptions; @@ -31,9 +32,14 @@ export class FlowrAnalyzer { this.controlflowInfo = undefined as unknown as ControlFlowInformation; } - public async normalizedAst(force?: boolean): Promise { + public async normalizedAst(force?: boolean): Promise { if(this.ast && !force) { - return this.ast; + return { + ...this.ast, + '.meta': { + cached: true + } + }; } const result = await createNormalizePipeline( @@ -44,9 +50,14 @@ export class FlowrAnalyzer { return result.normalize; } - public async dataflow(force?: boolean): Promise { + public async dataflow(force?: boolean): Promise { if(this.dataflowInfo && !force) { - return this.dataflowInfo; + return { + ...this.dataflowInfo, + '.meta': { + cached: true + } + }; } const result = await createDataflowPipeline( diff --git a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts index 432444f1013..9e9fc22f519 100644 --- a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts +++ b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts @@ -23,14 +23,14 @@ export function executeStaticSliceQuery({ dataflow: { graph }, ast, config }: Ba const slice = staticSlicing(graph, ast, criteria, config.solver.slicer?.threshold); const sliceEnd = Date.now(); if(noReconstruction) { - results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart } } }; + results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart, cached: false } } }; } else { const reconstructStart = Date.now(); const reconstruct = reconstructToCode(ast, slice.result, noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)); const reconstructEnd = Date.now(); results[key] = { - slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart } }, - reconstruct: { ...reconstruct, '.meta': { timing: reconstructEnd - reconstructStart } } + slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart, cached: false } }, + reconstruct: { ...reconstruct, '.meta': { timing: reconstructEnd - reconstructStart, cached: false } } }; } } From 30dc30c8189d75e0a19725ecf3d5f2d1966cbe14 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 22 Aug 2025 12:51:53 +0200 Subject: [PATCH 32/70] feat: use analyzer in repl query --- src/cli/repl/commands/repl-query.ts | 28 +++++++++---------- .../call-context-query-format.ts | 8 +++--- src/queries/query-print.ts | 28 +++++++++---------- src/queries/query.ts | 6 ++-- 4 files changed, 34 insertions(+), 36 deletions(-) diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index f207fff74be..a998641a4e2 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -1,5 +1,3 @@ -import type { DEFAULT_DATAFLOW_PIPELINE } from '../../../core/steps/pipeline/default-pipelines'; -import { createDataflowPipeline } from '../../../core/steps/pipeline/default-pipelines'; import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever'; import type { ReplCommand, ReplOutput } from './repl-main'; import { splitAtEscapeSensitive } from '../../../util/text/args'; @@ -7,19 +5,14 @@ import { ansiFormatter, italic } from '../../../util/text/ansi'; import { describeSchema } from '../../../util/schema'; import type { Query, QueryResults, SupportedQueryTypes } from '../../../queries/query'; import { AnyQuerySchema, executeQueries, QueriesSchema } from '../../../queries/query'; -import type { PipelineOutput } from '../../../core/steps/pipeline/pipeline'; import { jsonReplacer } from '../../../util/json'; import { asciiSummaryOfQueryResult } from '../../../queries/query-print'; import type { KnownParser } from '../../../r-bridge/parser'; import type { FlowrConfigOptions } from '../../../config'; import { getDummyFlowrProject } from '../../../project/flowr-project'; - - -async function getDataflow(config: FlowrConfigOptions, parser: KnownParser, remainingLine: string) { - return await createDataflowPipeline(parser, { - request: requestFromInput(remainingLine.trim()) - }, config).allRemainingSteps(); -} +import { FlowrAnalyzerBuilder } from '../../../project/flowr-analyzer-builder'; +import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { DataflowInformation } from '../../../dataflow/info'; function printHelp(output: ReplOutput) { @@ -33,7 +26,7 @@ function printHelp(output: ReplOutput) { output.stdout(`With this, ${italic(':query @config', output.formatter)} prints the result of the config query.`); } -async function processQueryArgs(line: string, parser: KnownParser, output: ReplOutput, config: FlowrConfigOptions): Promise, processed: PipelineOutput }> { +async function processQueryArgs(line: string, parser: KnownParser, output: ReplOutput, config: FlowrConfigOptions): Promise, processed: {dataflow: DataflowInformation, normalize: NormalizedAst} }> { const args = splitAtEscapeSensitive(line); const query = args.shift(); @@ -46,7 +39,7 @@ async function processQueryArgs(line: string, parser: KnownParser, output: ReplO return; } - let parsedQuery: Query[] = []; + let parsedQuery: Query[]; if(query.startsWith('@')) { parsedQuery = [{ type: query.slice(1) as SupportedQueryTypes } as Query]; const validationResult = QueriesSchema().validate(parsedQuery); @@ -69,10 +62,15 @@ async function processQueryArgs(line: string, parser: KnownParser, output: ReplO const dummyProject = await getDummyFlowrProject(); - const processed = await getDataflow(config, parser, args.join(' ')); + // TODO Use analyzer supplied by REPL core + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(args.join(' ').trim())) + .setParser(parser) + .setConfig(config) + .build(); + return { - query: executeQueries({ dataflow: processed.dataflow, ast: processed.normalize, config: config, libraries: dummyProject.libraries }, parsedQuery), - processed + query: executeQueries({ dataflow: await analyzer.dataflow(), ast: await analyzer.normalizedAst(), config: config, libraries: dummyProject.libraries }, parsedQuery), + processed: { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalizedAst() } }; } diff --git a/src/queries/catalog/call-context-query/call-context-query-format.ts b/src/queries/catalog/call-context-query/call-context-query-format.ts index c0865453f7a..d4fdfb85fe6 100644 --- a/src/queries/catalog/call-context-query/call-context-query-format.ts +++ b/src/queries/catalog/call-context-query/call-context-query-format.ts @@ -7,13 +7,13 @@ import { printAsMs } from '../../../util/text/time'; import Joi from 'joi'; import { asciiCallContext } from '../../query-print'; -import type { PipelineOutput } from '../../../core/steps/pipeline/pipeline'; -import type { DEFAULT_DATAFLOW_PIPELINE } from '../../../core/steps/pipeline/default-pipelines'; import { CallTargets } from './identify-link-to-last-call-relation'; import type { DataflowGraph } from '../../../dataflow/graph/graph'; import type { DataflowGraphVertexInfo } from '../../../dataflow/graph/vertex'; import type { CascadeAction } from './cascade-action'; import type { NoInfo } from '../../../r-bridge/lang-4.x/ast/model/model'; +import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { DataflowInformation } from '../../../dataflow/info'; export interface FileFilter { /** @@ -125,10 +125,10 @@ const CallContextQueryLinkTo = Joi.object({ export const CallContextQueryDefinition = { executor: executeCallContextQueries, - asciiSummarizer: (formatter: OutputFormatter, processed: PipelineOutput, queryResults: BaseQueryResult, result: string[]) => { + asciiSummarizer: (formatter: OutputFormatter, processed: {dataflow: DataflowInformation, normalize: NormalizedAst}, queryResults: BaseQueryResult, result: string[]) => { const out = queryResults as CallContextQueryResult; result.push(`Query: ${bold('call-context', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); - result.push(asciiCallContext(formatter, out, processed)); + result.push(asciiCallContext(formatter, out, processed.normalize.idMap)); return true; }, schema: Joi.object({ diff --git a/src/queries/query-print.ts b/src/queries/query-print.ts index ff6eabb951f..19ea4cdb3a6 100644 --- a/src/queries/query-print.ts +++ b/src/queries/query-print.ts @@ -1,60 +1,60 @@ import type { OutputFormatter } from '../util/text/ansi'; -import { markdownFormatter, bold, italic } from '../util/text/ansi'; +import { bold, italic, markdownFormatter } from '../util/text/ansi'; import type { QueryResults, SupportedQueryTypes } from './query'; import { SupportedQueries } from './query'; -import type { PipelineOutput } from '../core/steps/pipeline/pipeline'; -import type { DEFAULT_DATAFLOW_PIPELINE } from '../core/steps/pipeline/default-pipelines'; import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id'; import { textWithTooltip } from '../util/html-hover-over'; import type { CallContextQuerySubKindResult } from './catalog/call-context-query/call-context-query-format'; import type { BaseQueryMeta, BaseQueryResult } from './base-query-format'; import { printAsMs } from '../util/text/time'; import { isBuiltIn } from '../dataflow/environments/built-in'; +import type { AstIdMap, NormalizedAst, ParentInformation } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { DataflowInformation } from '../dataflow/info'; -function nodeString(nodeId: NodeId | { id: NodeId, info?: object}, formatter: OutputFormatter, processed: PipelineOutput): string { +function nodeString(nodeId: NodeId | { id: NodeId, info?: object}, formatter: OutputFormatter, idMap: AstIdMap): string { const isObj = typeof nodeId === 'object' && nodeId !== null && 'id' in nodeId; const id = isObj ? nodeId.id : nodeId; const info = isObj ? nodeId.info : undefined; if(isBuiltIn(id)) { return italic(id, formatter) + (info ? ` (${JSON.stringify(info)})` : ''); } - const node = processed.normalize.idMap.get(id); + const node = idMap.get(id); if(node === undefined) { return `UNKNOWN: ${id} (info: ${JSON.stringify(info)})`; } return `${italic('`' + (node.lexeme ?? node.info.fullLexeme ?? 'UNKNOWN') + '`', formatter)} (L.${node.location?.[0]}${info ? ', ' + JSON.stringify(info) : ''})`; } -function asciiCallContextSubHit(formatter: OutputFormatter, results: readonly CallContextQuerySubKindResult[], processed: PipelineOutput): string { +function asciiCallContextSubHit(formatter: OutputFormatter, results: readonly CallContextQuerySubKindResult[], idMap: AstIdMap): string { const result: string[] = []; for(const { id, calls = [], linkedIds = [], aliasRoots = [] } of results) { - const node = processed.normalize.idMap.get(id); + const node = idMap.get(id); if(node === undefined) { result.push(` ${bold('UNKNOWN: ' + JSON.stringify({ calls, linkedIds }))}`); continue; } - let line = nodeString(id, formatter, processed); + let line = nodeString(id, formatter, idMap); if(calls.length > 0) { - line += ` with ${calls.length} call${calls.length > 1 ? 's' : ''} (${calls.map(c => nodeString(c, formatter, processed)).join(', ')})`; + line += ` with ${calls.length} call${calls.length > 1 ? 's' : ''} (${calls.map(c => nodeString(c, formatter, idMap)).join(', ')})`; } if(linkedIds.length > 0) { - line += ` with ${linkedIds.length} link${linkedIds.length > 1 ? 's' : ''} (${linkedIds.map(c => nodeString(c, formatter, processed)).join(', ')})`; + line += ` with ${linkedIds.length} link${linkedIds.length > 1 ? 's' : ''} (${linkedIds.map(c => nodeString(c, formatter, idMap)).join(', ')})`; } if(aliasRoots.length > 0) { - line += ` with ${aliasRoots.length} alias root${aliasRoots.length > 1 ? 's' : ''} (${aliasRoots.map(c => nodeString(c, formatter, processed)).join(', ')})`; + line += ` with ${aliasRoots.length} alias root${aliasRoots.length > 1 ? 's' : ''} (${aliasRoots.map(c => nodeString(c, formatter, idMap)).join(', ')})`; } result.push(line); } return result.join(', '); } -export function asciiCallContext(formatter: OutputFormatter, results: QueryResults<'call-context'>['call-context'], processed: PipelineOutput): string { +export function asciiCallContext(formatter: OutputFormatter, results: QueryResults<'call-context'>['call-context'], idMap: AstIdMap): string { /* traverse over 'kinds' and within them 'subkinds' */ const result: string[] = []; for(const [kind, { subkinds }] of Object.entries(results['kinds'])) { result.push(` â•° ${bold(kind, formatter)}`); for(const [subkind, values] of Object.entries(subkinds)) { - result.push(` â•° ${bold(subkind, formatter)}: ${asciiCallContextSubHit(formatter, values, processed)}`); + result.push(` â•° ${bold(subkind, formatter)}: ${asciiCallContextSubHit(formatter, values, idMap)}`); } } return result.join('\n'); @@ -76,7 +76,7 @@ export function summarizeIdsIfTooLong(formatter: OutputFormatter, ids: readonly return formatter === markdownFormatter ? textWithTooltip(acc, JSON.stringify(ids)) : acc; } -export function asciiSummaryOfQueryResult(formatter: OutputFormatter, totalInMs: number, results: QueryResults, processed: PipelineOutput): string { +export function asciiSummaryOfQueryResult(formatter: OutputFormatter, totalInMs: number, results: QueryResults, processed: {dataflow: DataflowInformation, normalize: NormalizedAst}): string { const result: string[] = []; for(const [query, queryResults] of Object.entries(results)) { diff --git a/src/queries/query.ts b/src/queries/query.ts index 46988a1e37c..d602051e306 100644 --- a/src/queries/query.ts +++ b/src/queries/query.ts @@ -21,8 +21,6 @@ import { ClusterQueryDefinition } from './catalog/cluster-query/cluster-query-fo import type { DependenciesQuery } from './catalog/dependencies-query/dependencies-query-format'; import { DependenciesQueryDefinition } from './catalog/dependencies-query/dependencies-query-format'; import type { OutputFormatter } from '../util/text/ansi'; -import type { PipelineOutput } from '../core/steps/pipeline/pipeline'; -import type { DEFAULT_DATAFLOW_PIPELINE } from '../core/steps/pipeline/default-pipelines'; import Joi from 'joi'; import type { LocationMapQuery } from './catalog/location-map-query/location-map-query-format'; import { LocationMapQueryDefinition } from './catalog/location-map-query/location-map-query-format'; @@ -45,6 +43,8 @@ import { LinterQueryDefinition } from './catalog/linter-query/linter-query-forma import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id'; import type { ControlFlowQuery } from './catalog/control-flow-query/control-flow-query-format'; import { ControlFlowQueryDefinition } from './catalog/control-flow-query/control-flow-query-format'; +import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { DataflowInformation } from '../dataflow/info'; export type Query = CallContextQuery | ConfigQuery @@ -77,7 +77,7 @@ type SupportedQueries = { export interface SupportedQuery { executor: QueryExecutor, BaseQueryResult> - asciiSummarizer: (formatter: OutputFormatter, processed: PipelineOutput, queryResults: BaseQueryResult, resultStrings: string[]) => boolean + asciiSummarizer: (formatter: OutputFormatter, processed: {dataflow: DataflowInformation, normalize: NormalizedAst}, queryResults: BaseQueryResult, resultStrings: string[]) => boolean schema: Joi.ObjectSchema /** * Flattens the involved query nodes to be added to a flowR search when the {@link fromQuery} function is used based on the given result after this query is executed. From 4fc137e94557c17ed5abb8e3c3910149620ed39a Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Sat, 23 Aug 2025 09:23:41 +0200 Subject: [PATCH 33/70] feat: expose argument parser per command --- src/cli/repl/commands/repl-cfg.ts | 5 +++++ src/cli/repl/commands/repl-dataflow.ts | 5 +++++ src/cli/repl/commands/repl-main.ts | 2 ++ src/cli/repl/commands/repl-normalize.ts | 3 +++ 4 files changed, 15 insertions(+) diff --git a/src/cli/repl/commands/repl-cfg.ts b/src/cli/repl/commands/repl-cfg.ts index 288a2445ac9..f7bd1cbb5b7 100644 --- a/src/cli/repl/commands/repl-cfg.ts +++ b/src/cli/repl/commands/repl-cfg.ts @@ -7,6 +7,7 @@ import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/process import type { CfgSimplificationPassName } from '../../../control-flow/cfg-simplification'; import { DefaultCfgSimplificationOrder } from '../../../control-flow/cfg-simplification'; import type { FlowrAnalyzer } from '../../../project/flowr-analyzer'; +import { handleString } from '../core'; function formatInfo(out: ReplOutput, type: string): string { return out.formatter.format(`Copied ${type} to clipboard.`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); @@ -31,6 +32,7 @@ export const controlflowCommand: ReplCodeCommand = { usageExample: ':controlflow', aliases: [ 'cfg', 'cf' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { await produceAndPrintCfg(analyzer, output, [], cfgToMermaid); } @@ -43,6 +45,7 @@ export const controlflowStarCommand: ReplCodeCommand = { usageExample: ':controlflow*', aliases: [ 'cfg*', 'cf*' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { await produceAndPrintCfg(analyzer, output, [], cfgToMermaidUrl); } @@ -55,6 +58,7 @@ export const controlflowBbCommand: ReplCodeCommand = { usageExample: ':controlflowbb', aliases: [ 'cfgb', 'cfb' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { await produceAndPrintCfg(analyzer, output, ['to-basic-blocks'], cfgToMermaid); } @@ -67,6 +71,7 @@ export const controlflowBbStarCommand: ReplCodeCommand = { usageExample: ':controlflowbb*', aliases: [ 'cfgb*', 'cfb*' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { await produceAndPrintCfg(analyzer, output, ['to-basic-blocks' ], cfgToMermaidUrl); } diff --git a/src/cli/repl/commands/repl-dataflow.ts b/src/cli/repl/commands/repl-dataflow.ts index ca5b05e1282..3b8f023e8cd 100644 --- a/src/cli/repl/commands/repl-dataflow.ts +++ b/src/cli/repl/commands/repl-dataflow.ts @@ -3,6 +3,7 @@ import { fileProtocol } from '../../../r-bridge/retriever'; import { graphToMermaid, graphToMermaidUrl } from '../../../util/mermaid/dfg'; import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi'; import type { PipelinePerStepMetaInformation } from '../../../core/steps/pipeline/pipeline'; +import { handleString } from '../core'; function formatInfo(out: ReplOutput, type: string, meta: PipelinePerStepMetaInformation ): string { return out.formatter.format(`Copied ${type} to clipboard (dataflow: ${meta['.meta'].cached ? 'cached' : meta['.meta'].timing + 'ms'}).`, @@ -15,6 +16,7 @@ export const dataflowCommand: ReplCodeCommand = { usageExample: ':dataflow', aliases: [ 'd', 'df' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { const result = await analyzer.dataflow(); const mermaid = graphToMermaid({ graph: result.graph, includeEnvironments: false }).string; @@ -33,6 +35,7 @@ export const dataflowStarCommand: ReplCodeCommand = { usageExample: ':dataflow*', aliases: [ 'd*', 'df*' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { const result = await analyzer.dataflow(); const mermaid = graphToMermaidUrl(result.graph, false); @@ -52,6 +55,7 @@ export const dataflowSimplifiedCommand: ReplCodeCommand = { usageExample: ':dataflowsimple', aliases: [ 'ds', 'dfs' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { const result = await analyzer.dataflow(); const mermaid = graphToMermaid({ graph: result.graph, includeEnvironments: false, simplified: true }).string; @@ -70,6 +74,7 @@ export const dataflowSimpleStarCommand: ReplCodeCommand = { usageExample: ':dataflowsimple*', aliases: [ 'ds*', 'dfs*' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { const result = await analyzer.dataflow(); const mermaid = graphToMermaidUrl(result.graph, false, undefined, true); diff --git a/src/cli/repl/commands/repl-main.ts b/src/cli/repl/commands/repl-main.ts index c84c9784901..549c6f7c208 100644 --- a/src/cli/repl/commands/repl-main.ts +++ b/src/cli/repl/commands/repl-main.ts @@ -76,4 +76,6 @@ export interface ReplCodeCommand extends ReplBaseCommand { * Furthermore, it has to obey the formatter defined in the {@link ReplOutput}. */ fn: (info: ReplCodeCommandInformation) => Promise | void + /** Argument parser */ + argsParser: (remainingLine: string) => { input: string, remainingArgs: string[]} } diff --git a/src/cli/repl/commands/repl-normalize.ts b/src/cli/repl/commands/repl-normalize.ts index ae7e5b3b9cd..fff1bdf0a03 100644 --- a/src/cli/repl/commands/repl-normalize.ts +++ b/src/cli/repl/commands/repl-normalize.ts @@ -3,6 +3,7 @@ import { fileProtocol } from '../../../r-bridge/retriever'; import { normalizedAstToMermaid, normalizedAstToMermaidUrl } from '../../../util/mermaid/ast'; import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi'; import type { PipelinePerStepMetaInformation } from '../../../core/steps/pipeline/pipeline'; +import { handleString } from '../core'; function formatInfo(out: ReplOutput, type: string, meta: PipelinePerStepMetaInformation): string { return out.formatter.format(`Copied ${type} to clipboard (normalize: ${meta['.meta'].cached ? 'cached' : meta['.meta'].timing + 'ms'}).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); @@ -14,6 +15,7 @@ export const normalizeCommand: ReplCodeCommand = { usageExample: ':normalize', aliases: [ 'n' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { const result = await analyzer.normalizedAst(); const mermaid = normalizedAstToMermaid(result.ast); @@ -32,6 +34,7 @@ export const normalizeStarCommand: ReplCodeCommand = { usageExample: ':normalize*', aliases: [ 'n*' ], script: false, + argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { const result = await analyzer.normalizedAst(); const mermaid = normalizedAstToMermaidUrl(result.ast); From c955f10ff2d5b0774c60df58bb1db45fb2372d8a Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Sat, 23 Aug 2025 10:03:24 +0200 Subject: [PATCH 34/70] feat: pass analyzer to repl-query --- src/cli/repl/commands/repl-main.ts | 7 ++-- src/cli/repl/commands/repl-query.ts | 55 ++++++++++++++++------------- src/cli/repl/core.ts | 14 ++++---- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/cli/repl/commands/repl-main.ts b/src/cli/repl/commands/repl-main.ts index 549c6f7c208..34a8251ae4d 100644 --- a/src/cli/repl/commands/repl-main.ts +++ b/src/cli/repl/commands/repl-main.ts @@ -41,8 +41,9 @@ export interface ReplCommandInformation { } export interface ReplCodeCommandInformation { - output: ReplOutput, - analyzer: FlowrAnalyzer + output: ReplOutput, + analyzer: FlowrAnalyzer + remainingArgs: string[] } /** @@ -77,5 +78,5 @@ export interface ReplCodeCommand extends ReplBaseCommand { */ fn: (info: ReplCodeCommandInformation) => Promise | void /** Argument parser */ - argsParser: (remainingLine: string) => { input: string, remainingArgs: string[]} + argsParser: (remainingLine: string) => { input: string, remaining: string[]} } diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index a998641a4e2..bdd8faf167d 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -1,5 +1,5 @@ -import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever'; -import type { ReplCommand, ReplOutput } from './repl-main'; +import { fileProtocol } from '../../../r-bridge/retriever'; +import type { ReplCodeCommand, ReplOutput } from './repl-main'; import { splitAtEscapeSensitive } from '../../../util/text/args'; import { ansiFormatter, italic } from '../../../util/text/ansi'; import { describeSchema } from '../../../util/schema'; @@ -7,12 +7,10 @@ import type { Query, QueryResults, SupportedQueryTypes } from '../../../queries/ import { AnyQuerySchema, executeQueries, QueriesSchema } from '../../../queries/query'; import { jsonReplacer } from '../../../util/json'; import { asciiSummaryOfQueryResult } from '../../../queries/query-print'; -import type { KnownParser } from '../../../r-bridge/parser'; -import type { FlowrConfigOptions } from '../../../config'; import { getDummyFlowrProject } from '../../../project/flowr-project'; -import { FlowrAnalyzerBuilder } from '../../../project/flowr-analyzer-builder'; import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { DataflowInformation } from '../../../dataflow/info'; +import type { FlowrAnalyzer } from '../../../project/flowr-analyzer'; function printHelp(output: ReplOutput) { @@ -26,9 +24,8 @@ function printHelp(output: ReplOutput) { output.stdout(`With this, ${italic(':query @config', output.formatter)} prints the result of the config query.`); } -async function processQueryArgs(line: string, parser: KnownParser, output: ReplOutput, config: FlowrConfigOptions): Promise, processed: {dataflow: DataflowInformation, normalize: NormalizedAst} }> { - const args = splitAtEscapeSensitive(line); - const query = args.shift(); +async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalyzer, remainingArgs: string[]): Promise, processed: {dataflow: DataflowInformation, normalize: NormalizedAst} }> { + const query = remainingArgs[0]; if(!query) { output.stderr('No query provided, use \':query help\' to get more information.'); @@ -62,27 +59,36 @@ async function processQueryArgs(line: string, parser: KnownParser, output: ReplO const dummyProject = await getDummyFlowrProject(); - // TODO Use analyzer supplied by REPL core - const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(args.join(' ').trim())) - .setParser(parser) - .setConfig(config) - .build(); - return { - query: executeQueries({ dataflow: await analyzer.dataflow(), ast: await analyzer.normalizedAst(), config: config, libraries: dummyProject.libraries }, parsedQuery), + query: executeQueries({ + dataflow: await analyzer.dataflow(), + ast: await analyzer.normalizedAst(), + config: analyzer.flowrConfig, + libraries: dummyProject.libraries }, + parsedQuery), processed: { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalizedAst() } }; } -export const queryCommand: ReplCommand = { +function parseArgs(line: string) { + const args = splitAtEscapeSensitive(line); + const command = args.shift(); + return { + input: args.join(' ').trim(), + remaining: command ? [command] : [] + }; +} + +export const queryCommand: ReplCodeCommand = { description: `Query the given R code, start with '${fileProtocol}' to indicate a file. The query is to be a valid query in json format (use 'help' to get more information).`, - usesAnalyzer: false, + usesAnalyzer: true, usageExample: ':query "" ', aliases: [], script: false, - fn: async({ output, parser, remainingLine, config }) => { + argsParser: parseArgs, + fn: async({ output, analyzer, remainingArgs }) => { const totalStart = Date.now(); - const results = await processQueryArgs(remainingLine, parser, output, config); + const results = await processQueryArgs(output, analyzer, remainingArgs); const totalEnd = Date.now(); if(results) { output.stdout(asciiSummaryOfQueryResult(ansiFormatter, totalEnd - totalStart, results.query, results.processed)); @@ -90,14 +96,15 @@ export const queryCommand: ReplCommand = { } }; -export const queryStarCommand: ReplCommand = { +export const queryStarCommand: ReplCodeCommand = { description: 'Similar to query, but returns the output in json format.', - usesAnalyzer: false, + usesAnalyzer: true, usageExample: ':query* ', - aliases: [ ], + aliases: [], script: false, - fn: async({ output, parser, remainingLine, config }) => { - const results = await processQueryArgs(remainingLine, parser, output, config); + argsParser: parseArgs, + fn: async({ output, analyzer, remainingArgs }) => { + const results = await processQueryArgs(output, analyzer, remainingArgs); if(results) { output.stdout(JSON.stringify(results.query, jsonReplacer)); } diff --git a/src/cli/repl/core.ts b/src/cli/repl/core.ts index 41f00d5b606..ce2d766b300 100644 --- a/src/cli/repl/core.ts +++ b/src/cli/repl/core.ts @@ -81,8 +81,11 @@ export function makeDefaultReplReadline(): readline.ReadLineOptions { }; } -function handleString(code: string): string { - return code.startsWith('"') ? JSON.parse(code) as string : code; +export function handleString(code: string) { + return { + input: code.startsWith('"') ? JSON.parse(code) as string : code, + remaining: [] + }; } async function replProcessStatement(output: ReplOutput, statement: string, parser: KnownParser, allowRSessionAccess: boolean, config: FlowrConfigOptions): Promise { @@ -94,14 +97,13 @@ async function replProcessStatement(output: ReplOutput, statement: string, parse try { const remainingLine = statement.slice(command.length + 2).trim(); if(processor.usesAnalyzer) { - const request = requestFromInput(handleString(remainingLine)); - // TODO TSchoeller Ideally the analyzer would also be used for query commands - // TODO TSchoeller Is this the right level to create the analyzer instance? + const args = processor.argsParser(remainingLine); + const request = requestFromInput(args.input); const analyzer = await new FlowrAnalyzerBuilder(request) .setConfig(config) .setParser(parser) .build(); - await processor.fn({ output, analyzer }); + await processor.fn({ output, analyzer, remainingArgs: args.remaining }); } else { await processor.fn({ output, parser, remainingLine, allowRSessionAccess, config }); } From 034a92523d436f3fd856c9ca2d856790fa477c26 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Sat, 23 Aug 2025 18:14:52 +0200 Subject: [PATCH 35/70] feat: add parse pipeline to analyzer --- src/cli/repl/commands/repl-parse.ts | 31 ++++++++++++++------------- src/project/flowr-analyzer.ts | 33 +++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/cli/repl/commands/repl-parse.ts b/src/cli/repl/commands/repl-parse.ts index 06df3d8a531..cc9cf6a244a 100644 --- a/src/cli/repl/commands/repl-parse.ts +++ b/src/cli/repl/commands/repl-parse.ts @@ -1,11 +1,10 @@ -import type { ReplCommand } from './repl-main'; +import type { ReplCodeCommand } from './repl-main'; import type { OutputFormatter } from '../../../util/text/ansi'; import { FontStyles } from '../../../util/text/ansi'; import type { JsonEntry } from '../../../r-bridge/lang-4.x/ast/parser/json/format'; import { convertPreparedParsedData, prepareParsedData } from '../../../r-bridge/lang-4.x/ast/parser/json/format'; import { extractLocation, getTokenType, } from '../../../r-bridge/lang-4.x/ast/parser/main/normalize-meta'; -import { createParsePipeline } from '../../../core/steps/pipeline/default-pipelines'; -import { fileProtocol, removeRQuotes, requestFromInput } from '../../../r-bridge/retriever'; +import { fileProtocol, removeRQuotes } from '../../../r-bridge/retriever'; import type Parser from 'web-tree-sitter'; type DepthList = { depth: number, node: JsonEntry, leaf: boolean }[] @@ -153,25 +152,27 @@ function depthListToTextTree(list: Readonly, f: OutputFormatter): str return result; } -// TODO TSchoeller Should we use the analyzer here? -export const parseCommand: ReplCommand = { +export const parseCommand: ReplCodeCommand = { description: `Prints ASCII Art of the parsed, unmodified AST, start with '${fileProtocol}' to indicate a file`, - usesAnalyzer: false, + usesAnalyzer: true, usageExample: ':parse', aliases: [ 'p' ], script: false, - fn: async({ output, parser, remainingLine, config }) => { - const result = await createParsePipeline(parser, { - request: requestFromInput(removeRQuotes(remainingLine.trim())) - }, config).allRemainingSteps(); - - if(parser.name === 'r-shell') { - const object = convertPreparedParsedData(prepareParsedData(result.parse.parsed as unknown as string)); - + argsParser: (line: string) => { + return { + input: removeRQuotes(line.trim()), + remaining: [] + }; + }, + fn: async({ output, analyzer }) => { + const result = await analyzer.parseOutput(); + + if(analyzer.parserName() === 'r-shell') { + const object = convertPreparedParsedData(prepareParsedData(result.parsed as unknown as string)); output.stdout(depthListToTextTree(toDepthMap(object), output.formatter)); } else { // print the tree-sitter ast - output.stdout(depthListToTextTree(treeSitterToDepthList((result.parse.parsed as unknown as Parser.Tree).rootNode), output.formatter)); + output.stdout(depthListToTextTree(treeSitterToDepthList((result.parsed as unknown as Parser.Tree).rootNode), output.formatter)); } } }; diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 89c4eeda4c0..46a72eed9c5 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -1,7 +1,11 @@ import type { FlowrConfigOptions } from '../config'; import type { RParseRequests } from '../r-bridge/retriever'; -import { createDataflowPipeline, createNormalizePipeline } from '../core/steps/pipeline/default-pipelines'; -import type { KnownParser } from '../r-bridge/parser'; +import { + createDataflowPipeline, + createNormalizePipeline, + createParsePipeline +} from '../core/steps/pipeline/default-pipelines'; +import type { KnownParser, ParseStepOutput } from '../r-bridge/parser'; import type { Queries, SupportedQueryTypes } from '../queries/query'; import { executeQueries } from '../queries/query'; import { extractCfg } from '../control-flow/extract-cfg'; @@ -16,6 +20,7 @@ export class FlowrAnalyzer { private readonly request: RParseRequests; private readonly parser: KnownParser; + private parse = undefined as unknown as ParseStepOutput; private ast = undefined as unknown as NormalizedAst; private dataflowInfo = undefined as unknown as DataflowInformation; private controlflowInfo = undefined as unknown as ControlFlowInformation; @@ -32,6 +37,30 @@ export class FlowrAnalyzer { this.controlflowInfo = undefined as unknown as ControlFlowInformation; } + public parserName(): string { + return this.parser.name; + } + + // TODO TSchoeller Fix type + // TODO TSchoeller Do we want to expose parsing in this way? + public async parseOutput(force?: boolean): Promise & PipelinePerStepMetaInformation> { + if(this.parse && !force) { + return { + ...this.parse, + '.meta': { + cached: true + } + }; + } + + const result = await createParsePipeline( + this.parser, + { request: this.request }, + this.flowrConfig).allRemainingSteps(); + this.parse = result.parse; + return result.parse; + } + public async normalizedAst(force?: boolean): Promise { if(this.ast && !force) { return { From 1a100eaf91b40b30b3f6d232b5d352363e51c980 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:55:59 +0200 Subject: [PATCH 36/70] feat: introduce simpleCfg to analyzer --- src/cli/repl/commands/repl-cfg.ts | 2 +- src/project/flowr-analyzer.ts | 38 ++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/cli/repl/commands/repl-cfg.ts b/src/cli/repl/commands/repl-cfg.ts index f7bd1cbb5b7..f1230f8b656 100644 --- a/src/cli/repl/commands/repl-cfg.ts +++ b/src/cli/repl/commands/repl-cfg.ts @@ -14,7 +14,7 @@ function formatInfo(out: ReplOutput, type: string): string { } async function produceAndPrintCfg(analyzer: FlowrAnalyzer, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string) { - const cfg = await analyzer.controlflow([...DefaultCfgSimplificationOrder, ...simplifications]); + const cfg = await analyzer.controlFlow([...DefaultCfgSimplificationOrder, ...simplifications]); const result = await analyzer.normalizedAst(); const mermaid = cfgConverter(cfg, result); output.stdout(mermaid); diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 46a72eed9c5..dd331e225af 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -8,7 +8,7 @@ import { import type { KnownParser, ParseStepOutput } from '../r-bridge/parser'; import type { Queries, SupportedQueryTypes } from '../queries/query'; import { executeQueries } from '../queries/query'; -import { extractCfg } from '../control-flow/extract-cfg'; +import { extractCfg, extractSimpleCfg } from '../control-flow/extract-cfg'; import type { ControlFlowInformation } from '../control-flow/control-flow-graph'; import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { DataflowInformation } from '../dataflow/info'; @@ -23,7 +23,8 @@ export class FlowrAnalyzer { private parse = undefined as unknown as ParseStepOutput; private ast = undefined as unknown as NormalizedAst; private dataflowInfo = undefined as unknown as DataflowInformation; - private controlflowInfo = undefined as unknown as ControlFlowInformation; + private controlFlowInfo = undefined as unknown as ControlFlowInformation; + private simpleControlFlowInfo = undefined as unknown as ControlFlowInformation; // TODO Differentiate between simple and regular CFG constructor(config: FlowrConfigOptions, parser: KnownParser, request: RParseRequests) { this.flowrConfig = config; @@ -34,7 +35,7 @@ export class FlowrAnalyzer { public reset() { this.ast = undefined as unknown as NormalizedAst; this.dataflowInfo = undefined as unknown as DataflowInformation; - this.controlflowInfo = undefined as unknown as ControlFlowInformation; + this.controlFlowInfo = undefined as unknown as ControlFlowInformation; } public parserName(): string { @@ -98,17 +99,38 @@ export class FlowrAnalyzer { return result.dataflow; } - public async controlflow(simplifications?: readonly CfgSimplificationPassName[], force?: boolean): Promise { - if(this.controlflowInfo && !force) { - return this.controlflowInfo; + public async controlFlow(simplifications?: readonly CfgSimplificationPassName[], useDataflow?: boolean, force?: boolean): Promise { + if(this.controlFlowInfo && !force) { + return this.controlFlowInfo; } if(force || !this.ast) { await this.normalizedAst(force); } - const result = extractCfg(this.ast, this.flowrConfig); - this.controlflowInfo = result; + if(useDataflow && (force || !this.dataflowInfo)) { + await this.dataflow(force); + } + + const result = extractCfg(this.ast, this.flowrConfig, this.dataflowInfo.graph, simplifications); + this.controlFlowInfo = result; + return result; + } + + public async simpleControlFlow(simplifications?: readonly CfgSimplificationPassName[], force?: boolean): Promise { + if(this.simpleControlFlowInfo && !force) { + return this.simpleControlFlowInfo; + } else if(this.controlFlowInfo && !force) { + // Use the full CFG is it is already available + return this.controlFlowInfo; + } + + if(force || !this.ast) { + await this.normalizedAst(force); + } + + const result = extractSimpleCfg(this.ast); + this.simpleControlFlowInfo = result; return result; } From 1f5b9cb0b8e77128e7c5d3a450ee6e637dd602e5 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Thu, 28 Aug 2025 09:36:37 +0200 Subject: [PATCH 37/70] feat: use analyzer for server --- src/cli/repl/server/connection.ts | 137 +++++++++++++------------- src/project/flowr-analyzer.ts | 2 +- test/functionality/cli/server.test.ts | 19 ++-- 3 files changed, 78 insertions(+), 80 deletions(-) diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index c5526c01041..48de99d3533 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -20,38 +20,36 @@ import type { import { requestExecuteReplExpressionMessage } from './messages/message-repl'; import { replProcessAnswer } from '../core'; import { LogLevel } from '../../../util/log'; -import { cfg2quads, extractCfg } from '../../../control-flow/extract-cfg'; +import { cfg2quads } from '../../../control-flow/extract-cfg'; import type { QuadSerializationConfiguration } from '../../../util/quads'; import { defaultQuadIdGenerator } from '../../../util/quads'; import { printStepResult, StepOutputFormat } from '../../../core/print/print'; -import type { PARSE_WITH_R_SHELL_STEP } from '../../../core/steps/all/core/00-parse'; -import type { DataflowInformation } from '../../../dataflow/info'; -import type { NORMALIZE } from '../../../core/steps/all/core/10-normalize'; -import type { STATIC_DATAFLOW } from '../../../core/steps/all/core/20-dataflow'; +import { PARSE_WITH_R_SHELL_STEP } from '../../../core/steps/all/core/00-parse'; +import { NORMALIZE } from '../../../core/steps/all/core/10-normalize'; +import { STATIC_DATAFLOW } from '../../../core/steps/all/core/20-dataflow'; import { ansiFormatter, voidFormatter } from '../../../util/text/ansi'; -import { PipelineStepStage } from '../../../core/steps/pipeline-step'; -import { createSlicePipeline, DEFAULT_SLICING_PIPELINE } from '../../../core/steps/pipeline/default-pipelines'; -import type { Pipeline, PipelineOutput } from '../../../core/steps/pipeline/pipeline'; -import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; +import { DEFAULT_SLICING_PIPELINE } from '../../../core/steps/pipeline/default-pipelines'; +import type { PipelineOutput } from '../../../core/steps/pipeline/pipeline'; import type { DeepPartial } from 'ts-essentials'; import { DataflowGraph } from '../../../dataflow/graph/graph'; import * as tmp from 'tmp'; import fs from 'fs'; import type { RParseRequests } from '../../../r-bridge/retriever'; -import { makeMagicCommentHandler } from '../../../reconstruct/auto-select/magic-comments'; import type { LineageRequestMessage, LineageResponseMessage } from './messages/message-lineage'; import { requestLineageMessage } from './messages/message-lineage'; import { getLineage } from '../commands/repl-lineage'; -import { guard } from '../../../util/assert'; -import { doNotAutoSelect } from '../../../reconstruct/auto-select/auto-select-defaults'; import type { QueryRequestMessage, QueryResponseMessage } from './messages/message-query'; import { requestQueryMessage } from './messages/message-query'; import { executeQueries } from '../../../queries/query'; import type { KnownParser, ParseStepOutput } from '../../../r-bridge/parser'; -import type { PipelineExecutor } from '../../../core/pipeline-executor'; import { compact } from './compact'; import type { ControlFlowInformation } from '../../../control-flow/control-flow-graph'; import type { FlowrConfigOptions } from '../../../config'; +import { FlowrAnalyzerBuilder } from '../../../project/flowr-analyzer-builder'; +import type { FlowrAnalyzer } from '../../../project/flowr-analyzer'; +import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { DataflowInformation } from '../../../dataflow/info'; +import { PipelineStepStage } from '../../../core/steps/pipeline-step'; /** * Each connection handles a single client, answering to its requests. @@ -68,7 +66,7 @@ export class FlowRServerConnection { // maps token to information private readonly fileMap = new Map + analyzer: FlowrAnalyzer }>(); // we do not have to ensure synchronized shell-access as we are always running synchronized @@ -109,16 +107,16 @@ export class FlowRServerConnection { void this.handleFileAnalysisRequest(request.message as FileAnalysisRequestMessage); break; case 'request-slice': - this.handleSliceRequest(request.message as SliceRequestMessage); + void this.handleSliceRequest(request.message as SliceRequestMessage); break; case 'request-repl-execution': this.handleRepl(request.message as ExecuteRequestMessage); break; case 'request-lineage': - this.handleLineageRequest(request.message as LineageRequestMessage); + void this.handleLineageRequest(request.message as LineageRequestMessage); break; case 'request-query': - this.handleQueryRequest(request.message as QueryRequestMessage); + void this.handleQueryRequest(request.message as QueryRequestMessage); break; default: sendMessage(this.socket, { @@ -147,36 +145,32 @@ export class FlowRServerConnection { } const tempFile = tmp.fileSync({ postfix: '.R' }); - const slicer = this.createPipelineExecutorForRequest(message, tempFile.name); + const analyzer = await this.createAnalyzerForRequest(message, tempFile.name); - await slicer.allRemainingSteps(false).then(async results => await this.sendFileAnalysisResponse(slicer, results, message)) - .catch(e => { - this.logger.error(`[${this.name}] Error while analyzing file ${message.filename ?? 'unknown file'}: ${String(e)}`); - sendMessage(this.socket, { - id: message.id, - type: 'error', - fatal: false, - reason: `Error while analyzing file ${message.filename ?? 'unknown file'}: ${String(e)}` - }); + try { + await this.sendFileAnalysisResponse(analyzer, message); + } catch(e) { + this.logger.error(`[${this.name}] Error while analyzing file ${message.filename ?? 'unknown file'}: ${String(e)}`); + sendMessage(this.socket, { + id: message.id, + type: 'error', + fatal: false, + reason: `Error while analyzing file ${message.filename ?? 'unknown file'}: ${String(e)}` }); + } // this is an interestingly named function that means "I am a callback that removes a file" - so this deletes the file tempFile.removeCallback(); } - private async sendFileAnalysisResponse(slicer: PipelineExecutor, results: Partial>, message: FileAnalysisRequestMessage): Promise { + private async sendFileAnalysisResponse(analyzer: FlowrAnalyzer, message: FileAnalysisRequestMessage): Promise { let cfg: ControlFlowInformation | undefined = undefined; if(message.cfg) { - cfg = extractCfg(results.normalize as NormalizedAst, this.config, results.dataflow?.graph); + cfg = await analyzer.controlFlow(); } const config = (): QuadSerializationConfiguration => ({ context: message.filename ?? 'unknown', getId: defaultQuadIdGenerator() }); - const sanitizedResults = sanitizeAnalysisResults(results); - const pipeline = slicer.getPipeline(); - const parseStep = pipeline.steps.get('parse') as typeof PARSE_WITH_R_SHELL_STEP; - const normalizedStep = pipeline.steps.get('normalize') as typeof NORMALIZE; - const dataflowStep = pipeline.steps.get('dataflow') as typeof STATIC_DATAFLOW; - guard(parseStep !== undefined && normalizedStep !== undefined && dataflowStep !== undefined, 'All steps must be present'); + const sanitizedResults = sanitizeAnalysisResults(await analyzer.parseOutput(), await analyzer.normalizedAst(), await analyzer.dataflow()); if(message.format === 'n-quads') { sendMessage(this.socket, { type: 'response-file-analysis', @@ -184,9 +178,9 @@ export class FlowRServerConnection { id: message.id, cfg: cfg ? cfg2quads(cfg, config()) : undefined, results: { - parse: await printStepResult(parseStep, sanitizedResults.parse as ParseStepOutput, StepOutputFormat.RdfQuads, config()), - normalize: await printStepResult(normalizedStep, sanitizedResults.normalize as NormalizedAst, StepOutputFormat.RdfQuads, config()), - dataflow: await printStepResult(dataflowStep, sanitizedResults.dataflow as DataflowInformation, StepOutputFormat.RdfQuads, config()) + parse: await printStepResult(PARSE_WITH_R_SHELL_STEP, await analyzer.parseOutput(), StepOutputFormat.RdfQuads, config()), + normalize: await printStepResult(NORMALIZE, await analyzer.normalizedAst(), StepOutputFormat.RdfQuads, config()), + dataflow: await printStepResult(STATIC_DATAFLOW, await analyzer.dataflow(), StepOutputFormat.RdfQuads, config()) } }); } else if(message.format === 'compact') { @@ -208,7 +202,7 @@ export class FlowRServerConnection { } } - private createPipelineExecutorForRequest(message: FileAnalysisRequestMessage, tempFile: string) { + private async createAnalyzerForRequest(message: FileAnalysisRequestMessage, tempFile: string) { let request: RParseRequests; if(message.content !== undefined){ // we store the code in a temporary file in case it's too big for the shell to handle @@ -224,21 +218,22 @@ export class FlowRServerConnection { throw new Error('Either content or filepath must be defined.'); } - const slicer = createSlicePipeline(this.parser, { - request, - criterion: [] // currently unknown - }, this.config); + const analyzer = await new FlowrAnalyzerBuilder(request) + .setConfig(this.config) + .setParser(this.parser) + .build(); + if(message.filetoken) { this.logger.info(`Storing file token ${message.filetoken}`); this.fileMap.set(message.filetoken, { filename: message.filename, - pipeline: slicer + analyzer: analyzer }); } - return slicer; + return analyzer; } - private handleSliceRequest(base: SliceRequestMessage) { + private async handleSliceRequest(base: SliceRequestMessage) { const requestResult = validateMessage(base, requestSliceMessage); if(requestResult.type === 'error') { answerForValidationError(this.socket, requestResult, base.id); @@ -259,21 +254,25 @@ export class FlowRServerConnection { return; } - fileInformation.pipeline.updateRequest({ - criterion: request.criterion, - autoSelectIf: request.noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect) - }); + try { + // TODO TSchoeller Previously, this did update the slice request + const result = await fileInformation.analyzer.query([{ + type: 'static-slice', + criteria: request.criterion, + noMagicComments: request.noMagicComments, + // TODO TSchoeller autoSelectIf -> How to pass? + //autoSelectIf: request.noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect) + }]); - void fileInformation.pipeline.allRemainingSteps(true).then(results => { sendMessage(this.socket, { type: 'response-slice', id: request.id, results: Object.fromEntries( - Object.entries(results) + Object.entries(result) .filter(([k,]) => DEFAULT_SLICING_PIPELINE.steps.get(k)?.executed === PipelineStepStage.OncePerRequest) ) as SliceResponseMessage['results'] }); - }).catch(e => { + } catch(e) { this.logger.error(`[${this.name}] Error while analyzing file for token ${request.filetoken}: ${String(e)}`); sendMessage(this.socket, { id: request.id, @@ -281,7 +280,7 @@ export class FlowRServerConnection { fatal: false, reason: `Error while analyzing file for token ${request.filetoken}: ${String(e)}` }); - }); + } } @@ -318,7 +317,7 @@ export class FlowRServerConnection { }); } - private handleLineageRequest(base: LineageRequestMessage) { + private async handleLineageRequest(base: LineageRequestMessage) { const requestResult = validateMessage(base, requestLineageMessage); if(requestResult.type === 'error') { @@ -340,10 +339,8 @@ export class FlowRServerConnection { return; } - const { dataflow: dfg, normalize: ast } = fileInformation.pipeline.getResults(true); - guard(dfg !== undefined, `Dataflow graph must be present (request: ${request.filetoken})`); - guard(ast !== undefined, `AST must be present (request: ${request.filetoken})`); - const lineageIds = getLineage(request.criterion, dfg.graph, ast.idMap); + const analyzer = fileInformation.analyzer; + const lineageIds = getLineage(request.criterion, (await analyzer.dataflow()).graph, (await analyzer.normalizedAst()).idMap); sendMessage(this.socket, { type: 'response-lineage', id: request.id, @@ -351,7 +348,7 @@ export class FlowRServerConnection { }); } - private handleQueryRequest(base: QueryRequestMessage) { + private async handleQueryRequest(base: QueryRequestMessage) { const requestResult = validateMessage(base, requestQueryMessage); if(requestResult.type === 'error') { @@ -373,10 +370,11 @@ export class FlowRServerConnection { return; } - const { dataflow: dfg, normalize: ast } = fileInformation.pipeline.getResults(true); - guard(dfg !== undefined, `Dataflow graph must be present (request: ${request.filetoken})`); - guard(ast !== undefined, `AST must be present (request: ${request.filetoken})`); - const results = executeQueries({ dataflow: dfg, ast, config: this.config }, request.query); + const analyzer = fileInformation.analyzer; + const results = executeQueries({ + dataflow: await analyzer.dataflow(), + ast: await analyzer.normalizedAst(), + config: this.config }, request.query); sendMessage(this.socket, { type: 'response-query', id: request.id, @@ -385,17 +383,18 @@ export class FlowRServerConnection { } } -export function sanitizeAnalysisResults(results: Partial>): DeepPartial> { +export function sanitizeAnalysisResults(parse: ParseStepOutput, normalize: NormalizedAst, dataflow: DataflowInformation): DeepPartial> { return { - ...results, + // TODO Results missing? + parse, normalize: { - ...results.normalize, + ...normalize, idMap: undefined }, dataflow: { - ...results.dataflow, + ...dataflow, // we want to keep the DataflowGraph type information, but not the idMap - graph: new DataflowGraph(undefined).mergeWith(results.dataflow?.graph) + graph: new DataflowGraph(undefined).mergeWith(dataflow?.graph) } }; } diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index dd331e225af..2fe27e9d730 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -112,7 +112,7 @@ export class FlowrAnalyzer { await this.dataflow(force); } - const result = extractCfg(this.ast, this.flowrConfig, this.dataflowInfo.graph, simplifications); + const result = extractCfg(this.ast, this.flowrConfig, this.dataflowInfo?.graph, simplifications); this.controlFlowInfo = result; return result; } diff --git a/test/functionality/cli/server.test.ts b/test/functionality/cli/server.test.ts index 8e35fac22b9..0c4c7763feb 100644 --- a/test/functionality/cli/server.test.ts +++ b/test/functionality/cli/server.test.ts @@ -12,10 +12,8 @@ import type { FileAnalysisResponseMessageCompact, FileAnalysisResponseMessageJson } from '../../../src/cli/repl/server/messages/message-analysis'; -import { PipelineExecutor } from '../../../src/core/pipeline-executor'; import { jsonReplacer } from '../../../src/util/json'; import { extractCfg } from '../../../src/control-flow/extract-cfg'; -import { DEFAULT_DATAFLOW_PIPELINE } from '../../../src/core/steps/pipeline/default-pipelines'; import { requestFromInput } from '../../../src/r-bridge/retriever'; import { sanitizeAnalysisResults } from '../../../src/cli/repl/server/connection'; import type { QueryRequestMessage, QueryResponseMessage } from '../../../src/cli/repl/server/messages/message-query'; @@ -23,6 +21,7 @@ import { assert, describe, test } from 'vitest'; import { uncompact } from '../../../src/cli/repl/server/compact'; import { getPlatform } from '../../../src/util/os'; import { defaultConfigOptions } from '../../../src/config'; +import { FlowrAnalyzerBuilder } from '../../../src/project/flowr-analyzer-builder'; describe('flowr', () => { const skip = getPlatform() !== 'linux'; @@ -86,10 +85,10 @@ describe('flowr', () => { const response = messages[1] as FileAnalysisResponseMessageJson; // we are testing the server and not the slicer here! - const results = sanitizeAnalysisResults(await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, { - parser: shell, - request: requestFromInput('1 + 1'), - }, defaultConfigOptions).allRemainingSteps()); + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('1 + 1')) + .setParser(shell) + .build(); + const results = sanitizeAnalysisResults(await analyzer.parseOutput(), await analyzer.normalizedAst(), await analyzer.dataflow()); // cfg should not be set as we did not request it assert.isUndefined(response.cfg, 'Expected the cfg to be undefined as we did not request it'); @@ -120,10 +119,10 @@ describe('flowr', () => { const response = messages[1] as FileAnalysisResponseMessageCompact; // we are testing the server and not the slicer here! - const results = sanitizeAnalysisResults(await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, { - parser: shell, - request: requestFromInput('1 + 1'), - }, defaultConfigOptions).allRemainingSteps()); + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('1 + 1')) + .setParser(shell) + .build(); + const results = sanitizeAnalysisResults(await analyzer.parseOutput(), await analyzer.normalizedAst(), await analyzer.dataflow()); // cfg should not be set as we did not request it assert.isUndefined(response.cfg, 'Expected the cfg to be undefined as we did not request it'); From 2ddc051b91f9572c0595248dadd29710de341ff2 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Mon, 1 Sep 2025 08:46:25 +0200 Subject: [PATCH 38/70] feat: use analyzer for search, query, and linter --- src/cli/repl/commands/repl-query.ts | 6 +- src/cli/repl/server/connection.ts | 5 +- src/dataflow/graph/dataflowgraph-builder.ts | 33 +++++----- src/documentation/doc-util/doc-query.ts | 21 ++----- src/documentation/doc-util/doc-search.ts | 7 ++- src/linter/linter-executor.ts | 13 ++-- src/linter/linter-format.ts | 2 +- src/project/flowr-analyzer-builder.ts | 10 ++- src/project/flowr-analyzer.ts | 23 ++++--- src/queries/base-query-format.ts | 8 +-- .../call-context-query-executor.ts | 30 ++++++--- .../cluster-query/cluster-query-executor.ts | 4 +- .../config-query/config-query-executor.ts | 23 ++++--- .../control-flow-query-executor.ts | 4 +- .../dataflow-lens-query-executor.ts | 4 +- .../dataflow-query/dataflow-query-executor.ts | 4 +- .../dependencies-query-executor.ts | 35 ++++++++--- .../happens-before-query-executor.ts | 8 +-- .../id-map-query/id-map-query-executor.ts | 4 +- .../lineage-query/lineage-query-executor.ts | 9 +-- .../linter-query/linter-query-executor.ts | 5 +- .../location-map-query-executor.ts | 11 ++-- .../normalized-ast-query-executor.ts | 4 +- .../origin-query/origin-query-executor.ts | 10 +-- .../project-query/project-query-executor.ts | 4 +- .../resolve-value-query-executor.ts | 10 ++- .../search-query/search-query-executor.ts | 8 ++- .../static-slice-query-executor.ts | 6 +- .../static-slice-query-format.ts | 3 +- src/queries/query.ts | 18 +++--- src/search/flowr-search-executor.ts | 26 ++++---- src/search/flowr-search.ts | 15 ----- .../search-executor/search-enrichers.ts | 20 ++++-- .../search-executor/search-generators.ts | 40 ++++++------ src/search/search-executor/search-mappers.ts | 12 ++-- .../search-executor/search-transformer.ts | 62 +++++++++++-------- test/functionality/_helper/linter.ts | 40 +++++------- test/functionality/_helper/query.ts | 24 ++++--- test/functionality/_helper/search.ts | 52 ++++++++++------ test/functionality/_helper/shell.ts | 59 ++++++++++-------- ...ontainer-single-index-based-access.test.ts | 49 ++++++++------- test/functionality/slicing/slicing.bench.ts | 26 ++++---- 42 files changed, 409 insertions(+), 348 deletions(-) diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index bdd8faf167d..71692567bc5 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -60,10 +60,8 @@ async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalyzer, rem const dummyProject = await getDummyFlowrProject(); return { - query: executeQueries({ - dataflow: await analyzer.dataflow(), - ast: await analyzer.normalizedAst(), - config: analyzer.flowrConfig, + query: await executeQueries({ + input: analyzer, libraries: dummyProject.libraries }, parsedQuery), processed: { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalizedAst() } diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index 48de99d3533..a92b85dca74 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -371,10 +371,7 @@ export class FlowRServerConnection { } const analyzer = fileInformation.analyzer; - const results = executeQueries({ - dataflow: await analyzer.dataflow(), - ast: await analyzer.normalizedAst(), - config: this.config }, request.query); + const results = await executeQueries({ input: analyzer }, request.query); sendMessage(this.socket, { type: 'response-query', id: request.id, diff --git a/src/dataflow/graph/dataflowgraph-builder.ts b/src/dataflow/graph/dataflowgraph-builder.ts index f25dd65e7b7..8b88b9ebdf0 100644 --- a/src/dataflow/graph/dataflowgraph-builder.ts +++ b/src/dataflow/graph/dataflowgraph-builder.ts @@ -3,7 +3,7 @@ import type { NodeId } from '../../r-bridge/lang-4.x/ast/model/processing/node-i import { normalizeIdToNumberIfPossible } from '../../r-bridge/lang-4.x/ast/model/processing/node-id'; import type { AstIdMap } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { DataflowFunctionFlowInformation, FunctionArgument } from './graph'; -import { isPositionalArgument, DataflowGraph } from './graph'; +import { DataflowGraph, isPositionalArgument } from './graph'; import type { REnvironmentInformation } from '../environments/environment'; import { initializeCleanEnvironments } from '../environments/environment'; import type { DataflowGraphVertexAstLink, DataflowGraphVertexUse, FunctionOriginInformation } from './vertex'; @@ -16,9 +16,8 @@ import type { LinkTo } from '../../queries/catalog/call-context-query/call-conte import { DefaultBuiltinConfig, getDefaultProcessor } from '../environments/default-builtin-config'; import type { FlowrSearchLike } from '../../search/flowr-search-builder'; import { runSearch } from '../../search/flowr-search-executor'; -import type { Pipeline } from '../../core/steps/pipeline/pipeline'; -import type { FlowrSearchInput } from '../../search/flowr-search'; import { guard } from '../../util/assert'; +import type { FlowrAnalysisInput } from '../../project/flowr-analyzer'; export function emptyGraph(idMap?: AstIdMap) { return new DataflowGraphBuilder(idMap); @@ -204,12 +203,12 @@ export class DataflowGraphBuilder extends DataflowGraph { return this.addEdge(normalizeIdToNumberIfPossible(from), normalizeIdToNumberIfPossible(to as NodeId), type); } - private queryHelper(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput, type: EdgeType) { + private async queryHelper(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput, type: EdgeType) { let fromId: NodeId; if('nodeId' in from) { fromId = from.nodeId; } else { - const result = runSearch(from.query, data); + const result = await runSearch(from.query, data); guard(result.length === 1, `from query result should yield exactly one node, but yielded ${result.length}`); fromId = result[0].node.info.id; } @@ -218,7 +217,7 @@ export class DataflowGraphBuilder extends DataflowGraph { if('target' in to) { toIds = to.target; } else { - const result = runSearch(to.query, data); + const result = await runSearch(to.query, data); toIds = result.map(r => r.node.info.id); } @@ -241,10 +240,10 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @param from - Either a node id or a query to find the node id. * @param to - Either a node id or a query to find the node id. - * @param data - The data to search in i.e. the dataflow graph. + * @param input - The input to search in i.e. the dataflow graph. */ - public readsQuery

(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput

) { - return this.queryHelper(from, to, data, EdgeType.Reads); + public readsQuery(from: FromQueryParam, to: ToQueryParam, input: FlowrAnalysisInput) { + return this.queryHelper(from, to, input, EdgeType.Reads); } /** @@ -262,7 +261,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public definedByQuery

(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput

) { + public definedByQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { return this.queryHelper(from, to, data, EdgeType.DefinedBy); } @@ -280,7 +279,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public callsQuery

(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput

) { + public callsQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { return this.queryHelper(from, to, data, EdgeType.Calls); } @@ -298,7 +297,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public returnsQuery

(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput

) { + public returnsQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { return this.queryHelper(from, to, data, EdgeType.Returns); } @@ -316,7 +315,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public definesOnCallQuery

(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput

) { + public definesOnCallQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { return this.queryHelper(from, to, data, EdgeType.DefinesOnCall); } @@ -334,7 +333,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public definedByOnCallQuery

(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput

) { + public definedByOnCallQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { return this.queryHelper(from, to, data, EdgeType.DefinedByOnCall); } @@ -352,7 +351,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public argumentQuery

(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput

) { + public argumentQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { return this.queryHelper(from, to, data, EdgeType.Argument); } @@ -370,7 +369,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public nseQuery

(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput

) { + public nseQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { return this.queryHelper(from, to, data, EdgeType.NonStandardEvaluation); } @@ -388,7 +387,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public sideEffectOnCallQuery

(from: FromQueryParam, to: ToQueryParam, data: FlowrSearchInput

) { + public sideEffectOnCallQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { return this.queryHelper(from, to, data, EdgeType.SideEffectOnCall); } diff --git a/src/documentation/doc-util/doc-query.ts b/src/documentation/doc-util/doc-query.ts index a0d80b81d5b..7a77331865e 100644 --- a/src/documentation/doc-util/doc-query.ts +++ b/src/documentation/doc-util/doc-query.ts @@ -1,8 +1,6 @@ import type { RShell } from '../../r-bridge/shell'; import type { Queries, QueryResults, SupportedQueryTypes } from '../../queries/query'; import { executeQueries } from '../../queries/query'; -import { PipelineExecutor } from '../../core/pipeline-executor'; -import { DEFAULT_DATAFLOW_PIPELINE } from '../../core/steps/pipeline/default-pipelines'; import { requestFromInput } from '../../r-bridge/retriever'; import { jsonReplacer } from '../../util/json'; import { markdownFormatter } from '../../util/text/ansi'; @@ -13,26 +11,21 @@ import { printDfGraphForCode } from './doc-dfg'; import { codeBlock, jsonWithLimit } from './doc-code'; import { printAsMs } from '../../util/text/time'; import { asciiSummaryOfQueryResult } from '../../queries/query-print'; -import type { PipelineOutput } from '../../core/steps/pipeline/pipeline'; -import { defaultConfigOptions } from '../../config'; +import { FlowrAnalyzerBuilder } from '../../project/flowr-analyzer-builder'; -export interface ShowQueryOptions { +export interface ShowQueryOptions { readonly showCode?: boolean; readonly collapseResult?: boolean; readonly collapseQuery?: boolean; - readonly addOutput?: (result: QueryResults, pipeline: PipelineOutput) => string; } export async function showQuery< Base extends SupportedQueryTypes, VirtualArguments extends VirtualCompoundConstraint = VirtualCompoundConstraint ->(shell: RShell, code: string, queries: Queries, { showCode, collapseResult, collapseQuery, addOutput = () => '' }: ShowQueryOptions = {}): Promise { +>(shell: RShell, code: string, queries: Queries, { showCode, collapseResult, collapseQuery }: ShowQueryOptions = {}): Promise { const now = performance.now(); - const analysis = await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, { - parser: shell, - request: requestFromInput(code) - }, defaultConfigOptions).allRemainingSteps(); - const results = executeQueries({ dataflow: analysis.dataflow, ast: analysis.normalize, config: defaultConfigOptions }, queries); + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)).setParser(shell).build(); + const results = await executeQueries({ input: analyzer }, queries); const duration = performance.now() - now; const metaInfo = ` @@ -49,7 +42,7 @@ ${collapseResult ? '

Show Results, analysis) + asciiSummaryOfQueryResult(markdownFormatter, duration, results as QueryResults, { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalizedAst() }) }
Show Detailed Results as Json @@ -75,8 +68,6 @@ ${await printDfGraphForCode(shell, code, { switchCodeAndGraph: true })} ${collapseResult ? '
' : ''} -${addOutput(results, analysis)} - `; } diff --git a/src/documentation/doc-util/doc-search.ts b/src/documentation/doc-util/doc-search.ts index 66ffa89f52b..e7c3b26475b 100644 --- a/src/documentation/doc-util/doc-search.ts +++ b/src/documentation/doc-util/doc-search.ts @@ -14,6 +14,7 @@ import { flowrSearchToCode, flowrSearchToMermaid } from '../../search/flowr-sear import { recoverContent } from '../../r-bridge/lang-4.x/ast/model/processing/node-id'; import { formatRange } from '../../util/mermaid/dfg'; import { defaultConfigOptions } from '../../config'; +import { FlowrAnalyzerBuilder } from '../../project/flowr-analyzer-builder'; export interface ShowSearchOptions { readonly showCode?: boolean; @@ -26,7 +27,11 @@ export async function showSearch(shell: RShell, code: string, search: FlowrSearc parser: shell, request: requestFromInput(code) }, defaultConfigOptions).allRemainingSteps(); - const result = runSearch(search, { ...analysis, config: defaultConfigOptions }); + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) + .setParser(shell) + .setConfig(defaultConfigOptions) + .build(); + const result = await runSearch(search, analyzer); const duration = performance.now() - now; const metaInfo = ` diff --git a/src/linter/linter-executor.ts b/src/linter/linter-executor.ts index 6f241f10a97..33447641969 100644 --- a/src/linter/linter-executor.ts +++ b/src/linter/linter-executor.ts @@ -1,26 +1,25 @@ import type { LintingRuleConfig, LintingRuleMetadata, LintingRuleNames, LintingRuleResult } from './linter-rules'; import { LintingRules } from './linter-rules'; -import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; -import type { DataflowInformation } from '../dataflow/info'; import type { LintingResults, LintingRule } from './linter-format'; import { runSearch } from '../search/flowr-search-executor'; import { FlowrSearchElements } from '../search/flowr-search'; import type { DeepPartial } from 'ts-essentials'; import { deepMergeObject } from '../util/objects'; -import type { FlowrConfigOptions } from '../config'; +import type { FlowrAnalysisInput } from '../project/flowr-analyzer'; -export function executeLintingRule(ruleName: Name, input: { normalize: NormalizedAst, dataflow: DataflowInformation, config: FlowrConfigOptions }, lintingRuleConfig?: DeepPartial>): LintingResults { +export async function executeLintingRule(ruleName: Name, input: FlowrAnalysisInput, lintingRuleConfig?: DeepPartial>): Promise> { const rule = LintingRules[ruleName] as unknown as LintingRule, LintingRuleMetadata, LintingRuleConfig>; const fullConfig = deepMergeObject>(rule.info.defaultConfig, lintingRuleConfig); - const ruleSearch = rule.createSearch(fullConfig, input); + const ruleSearch = rule.createSearch(fullConfig); const searchStart = Date.now(); - const searchResult = runSearch(ruleSearch, input); + + const searchResult = await runSearch(ruleSearch, input); const searchTime = Date.now() - searchStart; const processStart = Date.now(); - const result = rule.processSearchResult(new FlowrSearchElements(searchResult), fullConfig, input); + const result = rule.processSearchResult(new FlowrSearchElements(searchResult), fullConfig, { normalize: await input.normalizedAst(), dataflow: await input.dataflow(), config: input.flowrConfig }); const processTime = Date.now() - processStart; return { diff --git a/src/linter/linter-format.ts b/src/linter/linter-format.ts index f2417c1a6d8..e7139bcc1ed 100644 --- a/src/linter/linter-format.ts +++ b/src/linter/linter-format.ts @@ -39,7 +39,7 @@ export interface LintingRule FlowrSearchLike> + readonly createSearch: (config: Config) => FlowrSearchLike> /** * Processes the search results of the search created through {@link createSearch}. * This function is expected to return the linting results from this rule for the given search, ie usually the given script file. diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index d96bb315758..3a7fc15f079 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -6,11 +6,13 @@ import { FlowrAnalyzer } from './flowr-analyzer'; import { retrieveEngineInstances } from '../engines'; import type { KnownParser } from '../r-bridge/parser'; import type { FlowrAnalyzerPlugin } from './plugins/flowr-analyzer-plugin'; +import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize'; export class FlowrAnalyzerBuilder { private flowrConfig: DeepWritable = cloneConfig(defaultConfigOptions); private parser?: KnownParser; private readonly request: RParseRequests; + private input?: Omit; private plugins: FlowrAnalyzerPlugin[]; public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions): this { @@ -33,6 +35,11 @@ export class FlowrAnalyzerBuilder { return this; } + public setInput(input: Omit) { + this.input = input; + return this; + } + constructor(request: RParseRequests, plugins?: FlowrAnalyzerPlugin[]) { this.request = request; this.plugins = plugins ?? []; @@ -56,7 +63,8 @@ export class FlowrAnalyzerBuilder { return new FlowrAnalyzer( this.flowrConfig, parser, - this.request + this.request, + this.input ?? {} ); } } \ No newline at end of file diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 2fe27e9d730..ea47bfdb078 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -14,11 +14,19 @@ import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/de import type { DataflowInformation } from '../dataflow/info'; import type { CfgSimplificationPassName } from '../control-flow/cfg-simplification'; import type { PipelinePerStepMetaInformation } from '../core/steps/pipeline/pipeline'; +import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize'; + +export type FlowrAnalysisInput = { + normalizedAst(force?: boolean): Promise; + dataflow(force?: boolean): Promise; + flowrConfig: FlowrConfigOptions; +} export class FlowrAnalyzer { - public readonly flowrConfig: FlowrConfigOptions; - private readonly request: RParseRequests; - private readonly parser: KnownParser; + public readonly flowrConfig: FlowrConfigOptions; + private readonly request: RParseRequests; + private readonly parser: KnownParser; + private readonly requiredInput: Omit; private parse = undefined as unknown as ParseStepOutput; private ast = undefined as unknown as NormalizedAst; @@ -26,10 +34,11 @@ export class FlowrAnalyzer { private controlFlowInfo = undefined as unknown as ControlFlowInformation; private simpleControlFlowInfo = undefined as unknown as ControlFlowInformation; // TODO Differentiate between simple and regular CFG - constructor(config: FlowrConfigOptions, parser: KnownParser, request: RParseRequests) { + constructor(config: FlowrConfigOptions, parser: KnownParser, request: RParseRequests, requiredInput: Omit) { this.flowrConfig = config; this.request = request; this.parser = parser; + this.requiredInput = requiredInput; } public reset() { @@ -43,7 +52,7 @@ export class FlowrAnalyzer { } // TODO TSchoeller Fix type - // TODO TSchoeller Do we want to expose parsing in this way? + // TODO TSchoeller Do we want to expose parsing as primary view in addition to the AST, CFG, and dataflow? public async parseOutput(force?: boolean): Promise & PipelinePerStepMetaInformation> { if(this.parse && !force) { return { @@ -74,7 +83,7 @@ export class FlowrAnalyzer { const result = await createNormalizePipeline( this.parser, - { request: this.request }, + { request: this.request, ...this.requiredInput }, this.flowrConfig).allRemainingSteps(); this.ast = result.normalize; return result.normalize; @@ -138,6 +147,6 @@ export class FlowrAnalyzer { if(!this.dataflowInfo) { await this.dataflow(force); } - return executeQueries({ ast: this.ast, dataflow: this.dataflowInfo, config: this.flowrConfig }, query); + return executeQueries({ input: this }, query); } } \ No newline at end of file diff --git a/src/queries/base-query-format.ts b/src/queries/base-query-format.ts index bc664d4983a..1316e0e80cd 100644 --- a/src/queries/base-query-format.ts +++ b/src/queries/base-query-format.ts @@ -1,7 +1,5 @@ -import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; -import type { DataflowInformation } from '../dataflow/info'; -import type { FlowrConfigOptions } from '../config'; import type { Package } from '../project/plugins/package-version-plugins/package'; +import type { FlowrAnalysisInput } from '../project/flowr-analyzer'; export interface BaseQueryFormat { /** used to select the query type :) */ @@ -18,7 +16,5 @@ export interface BaseQueryResult { export interface BasicQueryData { readonly libraries?: Package[]; - readonly ast: NormalizedAst; - readonly dataflow: DataflowInformation; - readonly config: FlowrConfigOptions; + readonly input: FlowrAnalysisInput; } diff --git a/src/queries/catalog/call-context-query/call-context-query-executor.ts b/src/queries/catalog/call-context-query/call-context-query-executor.ts index 3979a48e6e3..626e5c2d66a 100644 --- a/src/queries/catalog/call-context-query/call-context-query-executor.ts +++ b/src/queries/catalog/call-context-query/call-context-query-executor.ts @@ -200,7 +200,10 @@ function isParameterDefaultValue(nodeId: NodeId, ast: NormalizedAst): boolean { * This happens during the main resolution! * 4. Attach `linkTo` calls to the respective calls. */ -export function executeCallContextQueries({ dataflow: { graph }, ast, config }: BasicQueryData, queries: readonly CallContextQuery[]): CallContextQueryResult { +export async function executeCallContextQueries({ input }: BasicQueryData, queries: readonly CallContextQuery[]): Promise { + const ast = await input.normalizedAst(); + const dataflow = await input.dataflow(); + /* omit performance page load */ const now = Date.now(); /* the node id and call targets if present */ @@ -211,12 +214,12 @@ export function executeCallContextQueries({ dataflow: { graph }, ast, config }: let cfg = undefined; if(requiresCfg) { - cfg = extractCfg(ast, config, graph, []); + cfg = extractCfg(ast, input.flowrConfig, dataflow.graph, []); } const queriesWhichWantAliases = promotedQueries.filter(q => q.includeAliases); - for(const [nodeId, info] of graph.vertices(true)) { + for(const [nodeId, info] of dataflow.graph.vertices(true)) { if(info.tag !== VertexType.FunctionCall) { continue; } @@ -227,11 +230,15 @@ export function executeCallContextQueries({ dataflow: { graph }, ast, config }: * by checking all of these queries would be satisfied otherwise, * in general, we first want a call to happen, i.e., trace the called targets of this! */ - const targets = retrieveAllCallAliases(nodeId, graph); + const targets = retrieveAllCallAliases(nodeId, dataflow.graph); for(const [l, ids] of targets.entries()) { for(const query of queriesWhichWantAliases) { if(query.callName.test(l)) { - initialIdCollector.add(query.kind ?? '.', query.subkind ?? '.', compactRecord({ id: nodeId, name: info.name, aliasRoots: ids })); + initialIdCollector.add(query.kind ?? '.', query.subkind ?? '.', compactRecord({ + id: nodeId, + name: info.name, + aliasRoots: ids + })); } } } @@ -245,12 +252,12 @@ export function executeCallContextQueries({ dataflow: { graph }, ast, config }: let targets: NodeId[] | 'no' | undefined = undefined; if(query.callTargets) { - targets = satisfiesCallTargets(nodeId, graph, query.callTargets); + targets = satisfiesCallTargets(nodeId, dataflow.graph, query.callTargets); if(targets === 'no') { continue; } } - if(isQuoted(nodeId, graph)) { + if(isQuoted(nodeId, dataflow.graph)) { /* if the call is quoted, we do not want to link to it */ continue; } else if(query.ignoreParameterValues && isParameterDefaultValue(nodeId, ast)) { @@ -261,7 +268,7 @@ export function executeCallContextQueries({ dataflow: { graph }, ast, config }: const linked = Array.isArray(query.linkTo) ? query.linkTo : [query.linkTo]; for(const link of linked) { /* if we have a linkTo query, we have to find the last call */ - const lastCall = identifyLinkToLastCallRelation(nodeId, cfg.graph, graph, link); + const lastCall = identifyLinkToLastCallRelation(nodeId, cfg.graph, dataflow.graph, link); if(lastCall) { linkedIds ??= new Set(); for(const l of lastCall) { @@ -275,7 +282,12 @@ export function executeCallContextQueries({ dataflow: { graph }, ast, config }: } } - initialIdCollector.add(query.kind ?? '.', query.subkind ?? '.', compactRecord({ id: nodeId, name: info.name, calls: targets, linkedIds: linkedIds ? [...linkedIds] : undefined })); + initialIdCollector.add(query.kind ?? '.', query.subkind ?? '.', compactRecord({ + id: nodeId, + name: info.name, + calls: targets, + linkedIds: linkedIds ? [...linkedIds] : undefined + })); } } diff --git a/src/queries/catalog/cluster-query/cluster-query-executor.ts b/src/queries/catalog/cluster-query/cluster-query-executor.ts index d5955400cb6..4007dc4596e 100644 --- a/src/queries/catalog/cluster-query/cluster-query-executor.ts +++ b/src/queries/catalog/cluster-query/cluster-query-executor.ts @@ -4,13 +4,13 @@ import { findAllClusters } from '../../../dataflow/cluster'; import type { BasicQueryData } from '../../base-query-format'; -export function executeDataflowClusterQuery({ dataflow: { graph } }: BasicQueryData, queries: readonly DataflowClusterQuery[]): DataflowClusterQueryResult { +export async function executeDataflowClusterQuery({ input }: BasicQueryData, queries: readonly DataflowClusterQuery[]): Promise { if(queries.length !== 1) { log.warn('The dataflow cluster query expects only up to one query, but got', queries.length); } const start = Date.now(); - const clusters = findAllClusters(graph); + const clusters = findAllClusters((await input.dataflow()).graph); return { '.meta': { timing: Date.now() - start diff --git a/src/queries/catalog/config-query/config-query-executor.ts b/src/queries/catalog/config-query/config-query-executor.ts index c8fa3d107d1..713fb949e81 100644 --- a/src/queries/catalog/config-query/config-query-executor.ts +++ b/src/queries/catalog/config-query/config-query-executor.ts @@ -1,20 +1,19 @@ import { log } from '../../../util/log'; -import type { - ConfigQuery, - ConfigQueryResult -} from './config-query-format'; +import type { ConfigQuery, ConfigQueryResult } from './config-query-format'; import type { BasicQueryData } from '../../base-query-format'; -export function executeConfigQuery({ config }: BasicQueryData, queries: readonly ConfigQuery[]): ConfigQueryResult { +export function executeConfigQuery({ input }: BasicQueryData, queries: readonly ConfigQuery[]): Promise { if(queries.length !== 1) { log.warn('Config query expects only up to one query, but got', queries.length); } - return { - '.meta': { - /* there is no sense in measuring a get */ - timing: 0 - }, - config: config - }; + return new Promise(() => { + return { + '.meta': { + /* there is no sense in measuring a get */ + timing: 0 + }, + config: input.flowrConfig + }; + }); } diff --git a/src/queries/catalog/control-flow-query/control-flow-query-executor.ts b/src/queries/catalog/control-flow-query/control-flow-query-executor.ts index 67a418d2bcf..82deb06a91b 100644 --- a/src/queries/catalog/control-flow-query/control-flow-query-executor.ts +++ b/src/queries/catalog/control-flow-query/control-flow-query-executor.ts @@ -4,7 +4,7 @@ import type { BasicQueryData } from '../../base-query-format'; import { extractCfg } from '../../../control-flow/extract-cfg'; -export function executeControlFlowQuery({ dataflow: { graph }, ast, config }: BasicQueryData, queries: readonly ControlFlowQuery[]): ControlFlowQueryResult { +export async function executeControlFlowQuery({ input }: BasicQueryData, queries: readonly ControlFlowQuery[]): Promise { if(queries.length !== 1) { log.warn('The control flow query expects only up to one query, but got', queries.length); } @@ -12,7 +12,7 @@ export function executeControlFlowQuery({ dataflow: { graph }, ast, config }: Ba const query = queries[0]; const start = Date.now(); - const controlFlow = extractCfg(ast, config, graph, query.config?.simplificationPasses); + const controlFlow = extractCfg(await input.normalizedAst(), input.flowrConfig, (await input.dataflow()).graph, query.config?.simplificationPasses); return { '.meta': { timing: Date.now() - start diff --git a/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-executor.ts b/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-executor.ts index 679d2e2acd8..bb3a7c03d96 100644 --- a/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-executor.ts +++ b/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-executor.ts @@ -5,13 +5,13 @@ import { reduceDfg } from '../../../util/simple-df/dfg-view'; import { VertexType } from '../../../dataflow/graph/vertex'; -export function executeDataflowLensQuery({ dataflow: { graph } }: BasicQueryData, queries: readonly DataflowLensQuery[]): DataflowLensQueryResult { +export async function executeDataflowLensQuery({ input }: BasicQueryData, queries: readonly DataflowLensQuery[]): Promise { if(queries.length !== 1) { log.warn('Dataflow query expects only up to one query, but got', queries.length); } const now = Date.now(); - const simplifiedGraph = reduceDfg(graph, { + const simplifiedGraph = reduceDfg((await input.dataflow()).graph, { vertices: { keepEnv: false, keepCd: true, diff --git a/src/queries/catalog/dataflow-query/dataflow-query-executor.ts b/src/queries/catalog/dataflow-query/dataflow-query-executor.ts index efaeb63101b..17168c5569f 100644 --- a/src/queries/catalog/dataflow-query/dataflow-query-executor.ts +++ b/src/queries/catalog/dataflow-query/dataflow-query-executor.ts @@ -3,7 +3,7 @@ import { log } from '../../../util/log'; import type { BasicQueryData } from '../../base-query-format'; -export function executeDataflowQuery({ dataflow: { graph } }: BasicQueryData, queries: readonly DataflowQuery[]): DataflowQueryResult { +export async function executeDataflowQuery({ input }: BasicQueryData, queries: readonly DataflowQuery[]): Promise { if(queries.length !== 1) { log.warn('Dataflow query expects only up to one query, but got', queries.length); } @@ -12,6 +12,6 @@ export function executeDataflowQuery({ dataflow: { graph } }: BasicQueryData, qu /* there is no sense in measuring a get */ timing: 0 }, - graph + graph: (await input.dataflow()).graph }; } diff --git a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts index b5a346449e4..338d4050680 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts @@ -27,8 +27,11 @@ import { DependencyInfoLinkConstraint } from './function-info/function-info'; import { CallTargets } from '../call-context-query/identify-link-to-last-call-relation'; import { getArgumentStringValue } from '../../../dataflow/eval/resolve/resolve-argument'; import type { Package } from '../../../project/plugins/package-version-plugins/package'; +import type { DataflowInformation } from '../../../dataflow/info'; +import type { FlowrConfigOptions } from '../../../config'; +import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; -function collectNamespaceAccesses(data: BasicQueryData, libraries: LibraryInfo[]) { +function collectNamespaceAccesses(data: { ast: NormalizedAst, libraries?: Package[]}, libraries: LibraryInfo[]) { /* for libraries, we have to additionally track all uses of `::` and `:::`, for this we currently simply traverse all uses */ visitAst(data.ast.ast, n => { if(n.type === RType.Symbol && n.namespace) { @@ -44,10 +47,21 @@ function collectNamespaceAccesses(data: BasicQueryData, libraries: LibraryInfo[] }); } -export function executeDependenciesQuery(data: BasicQueryData, queries: readonly DependenciesQuery[]): DependenciesQueryResult { +export async function executeDependenciesQuery({ + input, + libraries +}: BasicQueryData, queries: readonly DependenciesQuery[]): Promise { if(queries.length !== 1) { log.warn('Dependencies query expects only up to one query, but got ', queries.length, 'only using the first query'); } + + const data = { + dataflow: await input.dataflow(), + ast: await input.normalizedAst(), + config: input.flowrConfig, + libraries: libraries + }; + const now = Date.now(); const [query] = queries; const ignoreDefault = query.ignoreDefaultFunctions ?? false; @@ -58,7 +72,10 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly const numberOfFunctions = libraryFunctions.length + sourceFunctions.length + readFunctions.length + writeFunctions.length; - const results = numberOfFunctions === 0 ? { kinds: {}, '.meta': { timing: 0 } } : executeQueriesOfSameType(data, + const results = numberOfFunctions === 0 ? { + kinds: {}, + '.meta': { timing: 0 } + } : await executeQueriesOfSameType({ input, libraries }, ...makeCallContextQuery(libraryFunctions, 'library'), ...makeCallContextQuery(sourceFunctions, 'source'), ...makeCallContextQuery(readFunctions, 'read'), @@ -76,7 +93,7 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly return get?.info.fullLexeme ?? get?.lexeme; } - const libraries: LibraryInfo[] = getResults(data, results, 'library', libraryFunctions, (id, vertex, argId, value, linkedIds) => ({ + const libs: LibraryInfo[] = getResults(data, results, 'library', libraryFunctions, (id, vertex, argId, value, linkedIds) => ({ nodeId: id, functionName: vertex.name, lexemeOfArgument: getLexeme(value, argId), @@ -87,7 +104,7 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly })); if(!ignoreDefault) { - collectNamespaceAccesses(data, libraries); + collectNamespaceAccesses(data, libs); } const sourcedFiles: SourceInfo[] = getResults(data, results, 'source', sourceFunctions, (id, vertex, argId, value, linkedIds) => ({ @@ -110,13 +127,13 @@ export function executeDependenciesQuery(data: BasicQueryData, queries: readonly // write functions that don't have argIndex are assumed to write to stdout destination: value ?? 'stdout', lexemeOfArgument: getLexeme(value, argId), - linkedIds: linkedIds?.length? linkedIds : undefined + linkedIds: linkedIds?.length ? linkedIds : undefined })); return { '.meta': { timing: Date.now() - now }, - libraries, sourcedFiles, readData, writtenData + libraries: libs, sourcedFiles, readData, writtenData }; } @@ -149,7 +166,7 @@ function dropInfoOnLinkedIds(linkedIds: readonly (NodeId | { id: NodeId, info: o return linkedIds.map(id => typeof id === 'object' ? id.id : id); } -function getResults(data: BasicQueryData, results: CallContextQueryResult, kind: string, functions: FunctionInfo[], makeInfo: MakeDependencyInfo): T[] { +function getResults(data: { dataflow: DataflowInformation, config: FlowrConfigOptions }, results: CallContextQueryResult, kind: string, functions: FunctionInfo[], makeInfo: MakeDependencyInfo): T[] { const kindEntries = Object.entries(results?.kinds[kind]?.subkinds ?? {}); return kindEntries.flatMap(([name, results]) => results.flatMap(({ id, linkedIds }) => { @@ -181,7 +198,7 @@ function getResults(data: BasicQueryData, results: Cal })) ?? []; } -function collectValuesFromLinks(args: Map> | undefined, data: BasicQueryData, linkedIds: readonly (NodeId | { id: NodeId, info: DependencyInfoLinkAttachedInfo })[] | undefined): Map> | undefined { +function collectValuesFromLinks(args: Map> | undefined, data: { dataflow: DataflowInformation, config: FlowrConfigOptions }, linkedIds: readonly (NodeId | { id: NodeId, info: DependencyInfoLinkAttachedInfo })[] | undefined): Map> | undefined { if(!linkedIds || linkedIds.length === 0) { return undefined; } diff --git a/src/queries/catalog/happens-before-query/happens-before-query-executor.ts b/src/queries/catalog/happens-before-query/happens-before-query-executor.ts index a93e64e9d1e..fe0e836ef55 100644 --- a/src/queries/catalog/happens-before-query/happens-before-query-executor.ts +++ b/src/queries/catalog/happens-before-query/happens-before-query-executor.ts @@ -1,17 +1,15 @@ import type { BasicQueryData } from '../../base-query-format'; -import type { - HappensBeforeQuery, - HappensBeforeQueryResult -} from './happens-before-query-format'; +import type { HappensBeforeQuery, HappensBeforeQueryResult } from './happens-before-query-format'; import { Ternary } from '../../../util/logic'; import { log } from '../../../util/log'; import { extractSimpleCfg } from '../../../control-flow/extract-cfg'; import { happensBefore } from '../../../control-flow/happens-before'; import { slicingCriterionToId } from '../../../slicing/criterion/parse'; -export function executeHappensBefore({ ast }: BasicQueryData, queries: readonly HappensBeforeQuery[]): HappensBeforeQueryResult { +export async function executeHappensBefore({ input }: BasicQueryData, queries: readonly HappensBeforeQuery[]): Promise { const start = Date.now(); const results: Record = {}; + const ast = await input.normalizedAst(); const cfg = extractSimpleCfg(ast); for(const query of queries) { const { a, b } = query; diff --git a/src/queries/catalog/id-map-query/id-map-query-executor.ts b/src/queries/catalog/id-map-query/id-map-query-executor.ts index d6c3125eec9..585c99bc4d7 100644 --- a/src/queries/catalog/id-map-query/id-map-query-executor.ts +++ b/src/queries/catalog/id-map-query/id-map-query-executor.ts @@ -2,7 +2,7 @@ import { log } from '../../../util/log'; import type { IdMapQuery, IdMapQueryResult } from './id-map-query-format'; import type { BasicQueryData } from '../../base-query-format'; -export function executeIdMapQuery({ ast }: BasicQueryData, queries: readonly IdMapQuery[]): IdMapQueryResult { +export async function executeIdMapQuery({ input }: BasicQueryData, queries: readonly IdMapQuery[]): Promise { if(queries.length !== 1) { log.warn('Id-Map query expects only up to one query, but got', queries.length); } @@ -12,6 +12,6 @@ export function executeIdMapQuery({ ast }: BasicQueryData, queries: readonly IdM /* there is no sense in measuring a get */ timing: 0 }, - idMap: ast.idMap + idMap: (await input.normalizedAst()).idMap }; } diff --git a/src/queries/catalog/lineage-query/lineage-query-executor.ts b/src/queries/catalog/lineage-query/lineage-query-executor.ts index e053cfdc41c..e914dfeeb33 100644 --- a/src/queries/catalog/lineage-query/lineage-query-executor.ts +++ b/src/queries/catalog/lineage-query/lineage-query-executor.ts @@ -1,19 +1,16 @@ -import type { - LineageQuery, - LineageQueryResult -} from './lineage-query-format'; +import type { LineageQuery, LineageQueryResult } from './lineage-query-format'; import { log } from '../../../util/log'; import { getLineage } from '../../../cli/repl/commands/repl-lineage'; import type { BasicQueryData } from '../../base-query-format'; -export function executeLineageQuery({ dataflow: { graph }, ast }: BasicQueryData, queries: readonly LineageQuery[]): LineageQueryResult { +export async function executeLineageQuery({ input }: BasicQueryData, queries: readonly LineageQuery[]): Promise { const start = Date.now(); const result: LineageQueryResult['lineages'] = {}; for(const { criterion } of queries) { if(result[criterion]) { log.warn('Duplicate criterion in lineage query:', criterion); } - result[criterion] = getLineage(criterion, graph, ast.idMap); + result[criterion] = getLineage(criterion, (await input.dataflow()).graph, (await input.normalizedAst()).idMap); } return { diff --git a/src/queries/catalog/linter-query/linter-query-executor.ts b/src/queries/catalog/linter-query/linter-query-executor.ts index e15213cce32..176fd6c5300 100644 --- a/src/queries/catalog/linter-query/linter-query-executor.ts +++ b/src/queries/catalog/linter-query/linter-query-executor.ts @@ -6,7 +6,7 @@ import { log } from '../../../util/log'; import type { ConfiguredLintingRule } from '../../../linter/linter-format'; import { executeLintingRule } from '../../../linter/linter-executor'; -export function executeLinterQuery({ ast, dataflow, config }: BasicQueryData, queries: readonly LinterQuery[]): LinterQueryResult { +export async function executeLinterQuery({ input }: BasicQueryData, queries: readonly LinterQuery[]): Promise { const flattened = queries.flatMap(q => q.rules ?? (Object.keys(LintingRules) as LintingRuleNames[])); const distinct = new Set(flattened); if(distinct.size !== flattened.length) { @@ -18,10 +18,9 @@ export function executeLinterQuery({ ast, dataflow, config }: BasicQueryData, qu const start = Date.now(); - const input = { normalize: ast, dataflow, config }; for(const entry of distinct) { const ruleName = typeof entry === 'string' ? entry : entry.name; - results.results[ruleName] = executeLintingRule(ruleName, input, (entry as ConfiguredLintingRule)?.config); + results.results[ruleName] = await executeLintingRule(ruleName, input, (entry as ConfiguredLintingRule)?.config); } return { diff --git a/src/queries/catalog/location-map-query/location-map-query-executor.ts b/src/queries/catalog/location-map-query/location-map-query-executor.ts index fb41b93b3a6..b2921bc99de 100644 --- a/src/queries/catalog/location-map-query/location-map-query-executor.ts +++ b/src/queries/catalog/location-map-query/location-map-query-executor.ts @@ -1,8 +1,5 @@ import { log } from '../../../util/log'; -import type { - LocationMapQuery, - LocationMapQueryResult -} from './location-map-query-format'; +import type { LocationMapQuery, LocationMapQueryResult } from './location-map-query-format'; import type { BasicQueryData } from '../../base-query-format'; import type { AstIdMap, RNodeWithParent } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; @@ -25,7 +22,7 @@ function fuzzyFindFile(node: RNodeWithParent | undefined, idMap: AstIdMap): stri return ''; } -export function executeLocationMapQuery({ ast, dataflow: { graph } }: BasicQueryData, queries: readonly LocationMapQuery[]): LocationMapQueryResult { +export async function executeLocationMapQuery({ input }: BasicQueryData, queries: readonly LocationMapQuery[]): Promise { if(queries.length !== 1) { log.warn('Id-Map query expects only up to one query, but got', queries.length); } @@ -36,11 +33,13 @@ export function executeLocationMapQuery({ ast, dataflow: { graph } }: BasicQuery }; let count = 0; const inverseMap = new Map(); - for(const file of graph.sourced) { + for(const file of (await input.dataflow()).graph.sourced) { locationMap.files[count] = file; inverseMap.set(file, count); count++; } + + const ast = await input.normalizedAst(); for(const [id, node] of ast.idMap.entries()) { if(node.location) { const file = fuzzyFindFile(node, ast.idMap); diff --git a/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts b/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts index 6d855e23499..f866fa1539e 100644 --- a/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts +++ b/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts @@ -3,7 +3,7 @@ import type { NormalizedAstQuery, NormalizedAstQueryResult } from './normalized- import type { BasicQueryData } from '../../base-query-format'; -export function executeNormalizedAstQuery({ ast }: BasicQueryData, queries: readonly NormalizedAstQuery[]): NormalizedAstQueryResult { +export async function executeNormalizedAstQuery({ input }: BasicQueryData, queries: readonly NormalizedAstQuery[]): Promise { if(queries.length !== 1) { log.warn('Normalized-Ast query expects only up to one query, but got', queries.length); } @@ -12,6 +12,6 @@ export function executeNormalizedAstQuery({ ast }: BasicQueryData, queries: read /* there is no sense in measuring a get */ timing: 0 }, - normalized: ast + normalized: await input.normalizedAst() }; } diff --git a/src/queries/catalog/origin-query/origin-query-executor.ts b/src/queries/catalog/origin-query/origin-query-executor.ts index c9653803451..c6ebd4c2f11 100644 --- a/src/queries/catalog/origin-query/origin-query-executor.ts +++ b/src/queries/catalog/origin-query/origin-query-executor.ts @@ -2,29 +2,29 @@ import { log } from '../../../util/log'; import type { BasicQueryData } from '../../base-query-format'; import type { SingleSlicingCriterion } from '../../../slicing/criterion/parse'; import { slicingCriterionToId } from '../../../slicing/criterion/parse'; -import type { OriginQueryResult, OriginQuery } from './origin-query-format'; +import type { OriginQuery, OriginQueryResult } from './origin-query-format'; import { getOriginInDfg } from '../../../dataflow/origin/dfg-get-origin'; export function fingerPrintOfQuery(query: OriginQuery): SingleSlicingCriterion { return query.criterion; } -export function executeResolveValueQuery({ dataflow: { graph }, ast }: BasicQueryData, queries: readonly OriginQuery[]): OriginQueryResult { +export async function executeResolveValueQuery({ input }: BasicQueryData, queries: readonly OriginQuery[]): Promise { const start = Date.now(); const results: OriginQueryResult['results'] = {}; for(const query of queries) { const key = fingerPrintOfQuery(query); - + if(results[key]) { log.warn(`Duplicate Key for origin-query: ${key}, skipping...`); } - const astId = slicingCriterionToId(key, ast.idMap); + const astId = slicingCriterionToId(key, (await input.normalizedAst()).idMap); if(astId === undefined) { log.warn(`Could not resolve id for ${key}, skipping...`); continue; } - results[key] = getOriginInDfg(graph, astId); + results[key] = getOriginInDfg((await input.dataflow()).graph, astId); } return { diff --git a/src/queries/catalog/project-query/project-query-executor.ts b/src/queries/catalog/project-query/project-query-executor.ts index dd08d93c071..0e116c43f03 100644 --- a/src/queries/catalog/project-query/project-query-executor.ts +++ b/src/queries/catalog/project-query/project-query-executor.ts @@ -2,7 +2,7 @@ import { log } from '../../../util/log'; import type { ProjectQuery, ProjectQueryResult } from './project-query-format'; import type { BasicQueryData } from '../../base-query-format'; -export function executeProjectQuery({ dataflow }: BasicQueryData, queries: readonly ProjectQuery[]): ProjectQueryResult { +export async function executeProjectQuery({ input }: BasicQueryData, queries: readonly ProjectQuery[]): Promise { if(queries.length !== 1) { log.warn('Project query expects only up to one query, but got', queries.length); } @@ -11,6 +11,6 @@ export function executeProjectQuery({ dataflow }: BasicQueryData, queries: reado /* there is no sense in measuring a get */ timing: 0 }, - files: dataflow.graph.sourced + files: (await input.dataflow()).graph.sourced }; } diff --git a/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts b/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts index 6eb75f2eb1d..88a3b14abf6 100644 --- a/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts +++ b/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts @@ -8,19 +8,23 @@ export function fingerPrintOfQuery(query: ResolveValueQuery): string { return JSON.stringify(query); } -export function executeResolveValueQuery({ dataflow: { graph }, ast, config }: BasicQueryData, queries: readonly ResolveValueQuery[]): ResolveValueQueryResult { +export async function executeResolveValueQuery({ input }: BasicQueryData, queries: readonly ResolveValueQuery[]): Promise { const start = Date.now(); const results: ResolveValueQueryResult['results'] = {}; + + const graph = (await input.dataflow()).graph; + const ast = await input.normalizedAst(); + for(const query of queries) { const key = fingerPrintOfQuery(query); - + if(results[key]) { log.warn(`Duplicate Key for resolve-value-query: ${key}, skipping...`); } const values = query.criteria .map(criteria => slicingCriterionToId(criteria, ast.idMap)) - .flatMap(ident => resolveIdToValue(ident, { graph, full: true, idMap: ast.idMap, resolve: config.solver.variables })); + .flatMap(ident => resolveIdToValue(ident, { graph, full: true, idMap: ast.idMap, resolve: input.flowrConfig.solver.variables })); results[key] = { values: values diff --git a/src/queries/catalog/search-query/search-query-executor.ts b/src/queries/catalog/search-query/search-query-executor.ts index e41af9ec9ec..944fb57b291 100644 --- a/src/queries/catalog/search-query/search-query-executor.ts +++ b/src/queries/catalog/search-query/search-query-executor.ts @@ -4,14 +4,16 @@ import { runSearch } from '../../../search/flowr-search-executor'; import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id'; import type { FlowrSearch } from '../../../search/flowr-search-builder'; -export function executeSearch({ ast, dataflow, config }: BasicQueryData, queries: readonly SearchQuery[]): SearchQueryResult { +export async function executeSearch({ input }: BasicQueryData, queries: readonly SearchQuery[]): Promise { const start = Date.now(); const results: { ids: NodeId[], search: FlowrSearch }[] = []; for(const query of queries) { const { search } = query; + + const searchResult = await runSearch(search, input); + results.push({ - ids: runSearch(search, { normalize: ast, dataflow, config } ) - .map(({ node }) => node.info.id), + ids: searchResult.map(({ node }) => node.info.id), search }); } diff --git a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts index 9e9fc22f519..c6cdbb55696 100644 --- a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts +++ b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts @@ -10,7 +10,7 @@ export function fingerPrintOfQuery(query: StaticSliceQuery): string { return JSON.stringify(query); } -export function executeStaticSliceQuery({ dataflow: { graph }, ast, config }: BasicQueryData, queries: readonly StaticSliceQuery[]): StaticSliceQueryResult { +export async function executeStaticSliceQuery({ input }: BasicQueryData, queries: readonly StaticSliceQuery[]): Promise { const start = Date.now(); const results: StaticSliceQueryResult['results'] = {}; for(const query of queries) { @@ -20,13 +20,13 @@ export function executeStaticSliceQuery({ dataflow: { graph }, ast, config }: Ba } const { criteria, noReconstruction, noMagicComments } = query; const sliceStart = Date.now(); - const slice = staticSlicing(graph, ast, criteria, config.solver.slicer?.threshold); + const slice = staticSlicing((await input.dataflow()).graph, await input.normalizedAst(), criteria, input.flowrConfig.solver.slicer?.threshold); const sliceEnd = Date.now(); if(noReconstruction) { results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart, cached: false } } }; } else { const reconstructStart = Date.now(); - const reconstruct = reconstructToCode(ast, slice.result, noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)); + const reconstruct = reconstructToCode(await input.normalizedAst(), slice.result, noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)); const reconstructEnd = Date.now(); results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart, cached: false } }, diff --git a/src/queries/catalog/static-slice-query/static-slice-query-format.ts b/src/queries/catalog/static-slice-query/static-slice-query-format.ts index 491e4e01f39..0cd2738da23 100644 --- a/src/queries/catalog/static-slice-query/static-slice-query-format.ts +++ b/src/queries/catalog/static-slice-query/static-slice-query-format.ts @@ -1,7 +1,8 @@ import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format'; import type { PipelineOutput } from '../../../core/steps/pipeline/pipeline'; import type { - DEFAULT_DATAFLOW_PIPELINE, DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE, + DEFAULT_DATAFLOW_PIPELINE, + DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE, DEFAULT_SLICING_PIPELINE } from '../../../core/steps/pipeline/default-pipelines'; import type { SlicingCriteria } from '../../../slicing/criterion/parse'; diff --git a/src/queries/query.ts b/src/queries/query.ts index d602051e306..fcd74f093da 100644 --- a/src/queries/query.ts +++ b/src/queries/query.ts @@ -69,14 +69,14 @@ export type Query = CallContextQuery export type QueryArgumentsWithType = Query & { type: QueryType }; /* Each executor receives all queries of its type in case it wants to avoid repeated traversal */ -export type QueryExecutor = (data: BasicQueryData, query: readonly Query[]) => Result; +export type QueryExecutor> = (data: BasicQueryData, query: readonly Query[]) => Result; type SupportedQueries = { [QueryType in Query['type']]: SupportedQuery } export interface SupportedQuery { - executor: QueryExecutor, BaseQueryResult> + executor: QueryExecutor, Promise> asciiSummarizer: (formatter: OutputFormatter, processed: {dataflow: DataflowInformation, normalize: NormalizedAst}, queryResults: BaseQueryResult, resultStrings: string[]) => boolean schema: Joi.ObjectSchema /** @@ -108,15 +108,16 @@ export const SupportedQueries = { } as const satisfies SupportedQueries; export type SupportedQueryTypes = keyof typeof SupportedQueries; -export type QueryResult = ReturnType; +export type QueryResult = Awaited>; -export function executeQueriesOfSameType(data: BasicQueryData, ...queries: readonly SpecificQuery[]): QueryResult { +export async function executeQueriesOfSameType(data: BasicQueryData, ...queries: readonly SpecificQuery[]): Promise> { guard(queries.length > 0, 'At least one query must be provided'); /* every query must have the same type */ guard(queries.every(q => q.type === queries[0].type), 'All queries must have the same type'); const query = SupportedQueries[queries[0].type]; guard(query !== undefined, `Unsupported query type: ${queries[0].type}`); - return query.executor(data, queries as never) as QueryResult; + const result = await query.executor(data, queries as never); + return result as QueryResult; } function isVirtualQuery< @@ -168,15 +169,16 @@ export type Queries< VirtualArguments extends VirtualCompoundConstraint = VirtualCompoundConstraint > = readonly (QueryArgumentsWithType | VirtualQueryArgumentsWithType)[]; -export function executeQueries< +export async function executeQueries< Base extends SupportedQueryTypes, VirtualArguments extends VirtualCompoundConstraint = VirtualCompoundConstraint ->(data: BasicQueryData, queries: Queries): QueryResults { +>(data: BasicQueryData, queries: Queries): Promise> { const now = Date.now(); const grouped = groupQueriesByType(queries); const results = {} as Writable>; for(const type of Object.keys(grouped) as Base[]) { - results[type] = executeQueriesOfSameType(data, ...grouped[type]) as QueryResults[Base]; + const result = await executeQueriesOfSameType(data, ...grouped[type]); + results[type] = result as QueryResults[Base]; } results['.meta'] = { timing: Date.now() - now diff --git a/src/search/flowr-search-executor.ts b/src/search/flowr-search-executor.ts index b92bb874f2c..b0835d362f8 100644 --- a/src/search/flowr-search-executor.ts +++ b/src/search/flowr-search-executor.ts @@ -1,12 +1,10 @@ import type { FlowrSearch, FlowrSearchLike, SearchOutput } from './flowr-search-builder'; import { getFlowrSearch } from './flowr-search-builder'; -import type { FlowrSearchElements, FlowrSearchInput } from './flowr-search'; - - -import type { Pipeline } from '../core/steps/pipeline/pipeline'; +import type { FlowrSearchElements } from './flowr-search'; import { getGenerator } from './search-executor/search-generators'; import { getTransformer } from './search-executor/search-transformer'; import type { ParentInformation } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { FlowrAnalysisInput } from '../project/flowr-analyzer'; type GetSearchElements = S extends FlowrSearch ? Elements extends FlowrSearchElements ? E : never : never; @@ -14,15 +12,17 @@ type GetSearchElements = S extends FlowrSearch( +export async function runSearch( search: S, - data: FlowrSearchInput

-): GetSearchElements> { + input: FlowrAnalysisInput +): Promise>> { const s = getFlowrSearch(search); - return s.search.reduce( - (acc: FlowrSearchElements, transformer) => - getTransformer(transformer.name)(data, acc, transformer.args as never), - /* support multiple arguments may be abstracted away in search frontend */ - getGenerator(s.generator.name)(data, s.generator.args as never) - ).getElements() as GetSearchElements>; + + let acc: FlowrSearchElements = await getGenerator(s.generator.name)(input, s.generator.args as never); + + for(const transformer of s.search) { + acc = await getTransformer(transformer.name)(input, acc, transformer.args as never); + } + + return acc.getElements() as GetSearchElements>; } diff --git a/src/search/flowr-search.ts b/src/search/flowr-search.ts index 6fe1bbd9c0a..ec6b3cfc444 100644 --- a/src/search/flowr-search.ts +++ b/src/search/flowr-search.ts @@ -1,11 +1,7 @@ import type { NoInfo, RNode } from '../r-bridge/lang-4.x/ast/model/model'; -import type { Pipeline, PipelineOutput, PipelineStepOutputWithName } from '../core/steps/pipeline/pipeline'; -import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id'; -import type { DataflowInformation } from '../dataflow/info'; import type { BaseQueryResult } from '../queries/base-query-format'; import type { Query } from '../queries/query'; -import type { FlowrConfigOptions } from '../config'; import type { MarkOptional } from 'ts-essentials'; /** @@ -58,17 +54,6 @@ export interface FlowrSearchGetFilter extends Record { readonly id?: NodeId; } -type MinimumInputForFlowrSearch

= - PipelineStepOutputWithName extends NormalizedAst ? ( - PipelineStepOutputWithName extends DataflowInformation ? PipelineOutput

& { normalize: NormalizedAst, dataflow: DataflowInformation, config: FlowrConfigOptions } - : never - ): never - -/** we allow any pipeline, which provides us with a 'normalize' and 'dataflow' step */ -export type FlowrSearchInput< - P extends Pipeline -> = MinimumInputForFlowrSearch

- /** Intentionally, we abstract away from an array to avoid the use of conventional typescript operations */ export class FlowrSearchElements[] = FlowrSearchElement[]> { private elements: Elements = [] as unknown as Elements; diff --git a/src/search/search-executor/search-enrichers.ts b/src/search/search-executor/search-enrichers.ts index ec605af1a7b..680e3e0d6e2 100644 --- a/src/search/search-executor/search-enrichers.ts +++ b/src/search/search-executor/search-enrichers.ts @@ -1,6 +1,9 @@ -import type { FlowrSearchElement, FlowrSearchInput } from '../flowr-search'; -import type { ParentInformation, RNodeWithParent } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; -import type { Pipeline } from '../../core/steps/pipeline/pipeline'; +import type { FlowrSearchElement } from '../flowr-search'; +import type { + NormalizedAst, + ParentInformation, + RNodeWithParent +} from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { MergeableRecord } from '../../util/objects'; import { VertexType } from '../../dataflow/graph/vertex'; import type { Identifier } from '../../dataflow/environments/identifier'; @@ -12,6 +15,7 @@ import { guard, isNotUndefined } from '../../util/assert'; import { extractSimpleCfg } from '../../control-flow/extract-cfg'; import { getOriginInDfg, OriginType } from '../../dataflow/origin/dfg-get-origin'; import { recoverName } from '../../r-bridge/lang-4.x/ast/model/processing/node-id'; +import type { DataflowInformation } from '../../dataflow/info'; /** * A {@link FlowrSearchElement} that is enriched with a set of enrichments through {@link FlowrSearchBuilder.with}. @@ -21,11 +25,16 @@ export interface EnrichedFlowrSearchElement extends FlowrSearchElement } } +export interface EnrichmentInput { + dataflow: DataflowInformation, + normalize: NormalizedAst +} + export interface EnrichmentData { /** * A function that is applied to each element of the search to enrich it with additional data. */ - readonly enrich: (e: FlowrSearchElement, data: FlowrSearchInput, args: EnrichmentArguments | undefined) => EnrichmentContent + readonly enrich: (e: FlowrSearchElement, input: EnrichmentInput, args: EnrichmentArguments | undefined) => EnrichmentContent /** * The mapping function used by the {@link Mapper.Enrichment} mapper. */ @@ -132,8 +141,9 @@ export function enrich< ElementIn extends FlowrSearchElement, ElementOut extends ElementIn & EnrichedFlowrSearchElement, ConcreteEnrichment extends Enrichment>( - e: ElementIn, data: FlowrSearchInput, enrichment: ConcreteEnrichment, args?: EnrichmentArguments): ElementOut { + e: ElementIn, data: EnrichmentInput, enrichment: ConcreteEnrichment, args?: EnrichmentArguments): ElementOut { const enrichmentData = Enrichments[enrichment] as unknown as EnrichmentData, EnrichmentArguments>; + return { ...e, enrichments: { diff --git a/src/search/search-executor/search-generators.ts b/src/search/search-executor/search-generators.ts index 728871ba7bb..944d7122c2d 100644 --- a/src/search/search-executor/search-generators.ts +++ b/src/search/search-executor/search-generators.ts @@ -2,11 +2,9 @@ import type { FlowrSearchElement, FlowrSearchElementFromQuery, FlowrSearchGeneratorNodeBase, - FlowrSearchGetFilter, - FlowrSearchInput + FlowrSearchGetFilter } from '../flowr-search'; import { FlowrSearchElements } from '../flowr-search'; -import type { Pipeline } from '../../core/steps/pipeline/pipeline'; import type { TailTypesOrUndefined } from '../../util/collections/arrays'; import type { ParentInformation, RNodeWithParent } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { SlicingCriteria } from '../../slicing/criterion/parse'; @@ -16,6 +14,7 @@ import type { Query, SupportedQuery } from '../../queries/query'; import { executeQueries, SupportedQueries } from '../../queries/query'; import type { BaseQueryResult } from '../../queries/base-query-format'; import type { RNode } from '../../r-bridge/lang-4.x/ast/model/model'; +import type { FlowrAnalysisInput } from '../../project/flowr-analyzer'; /** * This is a union of all possible generator node types @@ -43,26 +42,28 @@ export const generators = { 'from-query': generateFromQuery } as const; -function generateAll(data: FlowrSearchInput): FlowrSearchElements { - return new FlowrSearchElements(getAllNodes(data) +async function generateAll(data: FlowrAnalysisInput): Promise> { + return new FlowrSearchElements((await getAllNodes(data)) .map(node => ({ node }))); } -function getAllNodes(data: FlowrSearchInput): RNodeWithParent[] { - return [...new Map([... data.normalize.idMap.values()].map(n => [n.info.id, n])) +async function getAllNodes(data: FlowrAnalysisInput): Promise { + const normalize = await data.normalizedAst(); + return [...new Map([...normalize.idMap.values()].map(n => [n.info.id, n])) .values()]; } -function generateGet(data: FlowrSearchInput, { filter: { line, column, id, name, nameIsRegex } }: { filter: FlowrSearchGetFilter }): FlowrSearchElements { +async function generateGet(input: FlowrAnalysisInput, { filter: { line, column, id, name, nameIsRegex } }: { filter: FlowrSearchGetFilter }): Promise> { + const normalize = await input.normalizedAst(); let potentials = (id ? - [data.normalize.idMap.get(id)].filter(isNotUndefined) : - getAllNodes(data) + [normalize.idMap.get(id)].filter(isNotUndefined) : + await getAllNodes(input) ); if(line && line < 0) { - const maxLines = data.normalize.ast.info.fullRange?.[2] ?? - (id ? getAllNodes(data) : potentials).reduce( + const maxLines = normalize.ast.info.fullRange?.[2] ?? + (id ? (await getAllNodes(input)) : potentials).reduce( (maxLine, { location }) => location && location[2] > maxLine ? location[2] : maxLine, 0 ); @@ -86,13 +87,15 @@ function generateGet(data: FlowrSearchInput, { filter: { line, column, return new FlowrSearchElements(potentials.map(node => ({ node }))); } -function generateFrom(data: FlowrSearchInput, args: { from: FlowrSearchElement | FlowrSearchElement[] }): FlowrSearchElements { +function generateFrom(_input: FlowrAnalysisInput, args: { from: FlowrSearchElement | FlowrSearchElement[] }): FlowrSearchElements { return new FlowrSearchElements(Array.isArray(args.from) ? args.from : [args.from]); } -function generateFromQuery(data: FlowrSearchInput, args: { from: readonly Query[] } ): FlowrSearchElements[]> { +async function generateFromQuery(input: FlowrAnalysisInput, args: { + from: readonly Query[] +}): Promise[]>> { const nodes = new Set>(); - const result = executeQueries({ ast: data.normalize, dataflow: data.dataflow, config: data.config }, args.from); + const result = await executeQueries({ input }, args.from); for(const [query, content] of Object.entries(result)) { if(query === '.meta') { continue; @@ -100,7 +103,7 @@ function generateFromQuery(data: FlowrSearchInput, args: { from: reado const queryDef = SupportedQueries[query as Query['type']] as SupportedQuery; for(const node of queryDef.flattenInvolvedNodes(content as BaseQueryResult)) { nodes.add({ - node: data.normalize.idMap.get(node) as RNode, + node: (await input.normalizedAst()).idMap.get(node) as RNode, query: query as Query['type'], queryResult: content as BaseQueryResult }); @@ -109,9 +112,10 @@ function generateFromQuery(data: FlowrSearchInput, args: { from: reado return new FlowrSearchElements([...nodes]); } -function generateCriterion(data: FlowrSearchInput, args: { criterion: SlicingCriteria }): FlowrSearchElements { +async function generateCriterion(input: FlowrAnalysisInput, args: { criterion: SlicingCriteria }): Promise> { + const idMap = (await input.normalizedAst()).idMap; return new FlowrSearchElements( - args.criterion.map(c => ({ node: data.normalize.idMap.get(slicingCriterionToId(c, data.normalize.idMap)) as RNodeWithParent })) + args.criterion.map(c => ({ node: idMap.get(slicingCriterionToId(c, idMap)) as RNodeWithParent })) ); } diff --git a/src/search/search-executor/search-mappers.ts b/src/search/search-executor/search-mappers.ts index 98c2c2dab38..2cd99d8ec31 100644 --- a/src/search/search-executor/search-mappers.ts +++ b/src/search/search-executor/search-mappers.ts @@ -1,23 +1,23 @@ -import type { FlowrSearchElement, FlowrSearchInput } from '../flowr-search'; +import type { FlowrSearchElement } from '../flowr-search'; import type { ParentInformation } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { Enrichment } from './search-enrichers'; -import { enrichmentContent , Enrichments } from './search-enrichers'; +import { enrichmentContent, Enrichments } from './search-enrichers'; import type { MergeableRecord } from '../../util/objects'; -import type { Pipeline } from '../../core/steps/pipeline/pipeline'; +import type { FlowrAnalysisInput } from '../../project/flowr-analyzer'; export enum Mapper { Enrichment = 'enrichment' } export interface MapperData { - mapper: (e: FlowrSearchElement, data: FlowrSearchInput, args: Arguments) => FlowrSearchElement[] + mapper: (e: FlowrSearchElement, data: FlowrAnalysisInput, args: Arguments) => FlowrSearchElement[] } export type MapperArguments = typeof Mappers[M] extends MapperData ? Arguments : never; const Mappers = { [Mapper.Enrichment]: { - mapper: (e: FlowrSearchElement, _data: FlowrSearchInput, enrichment: Enrichment) => { + mapper: (e: FlowrSearchElement, _data: FlowrAnalysisInput, enrichment: Enrichment) => { const data = enrichmentContent(e, enrichment); return data !== undefined ? Enrichments[enrichment].mapper(data) : []; } @@ -25,6 +25,6 @@ const Mappers = { } as const; export function map, MapperType extends Mapper>( - e: Element, data: FlowrSearchInput, mapper: MapperType, args: MapperArguments): Element[] { + e: Element, data: FlowrAnalysisInput, mapper: MapperType, args: MapperArguments): Element[] { return (Mappers[mapper] as MapperData>).mapper(e, data, args) as Element[]; } diff --git a/src/search/search-executor/search-transformer.ts b/src/search/search-executor/search-transformer.ts index e187ac1c810..0b41b1ad492 100644 --- a/src/search/search-executor/search-transformer.ts +++ b/src/search/search-executor/search-transformer.ts @@ -1,12 +1,4 @@ -import type { - FlowrSearchElement, - FlowrSearchElements, - FlowrSearchInput, - FlowrSearchTransformerNodeBase -} from '../flowr-search'; - - -import type { Pipeline } from '../../core/steps/pipeline/pipeline'; +import type { FlowrSearchElement, FlowrSearchElements, FlowrSearchTransformerNodeBase } from '../flowr-search'; import type { LastOfArray, Tail2TypesOrUndefined, TailOfArray } from '../../util/collections/arrays'; import type { FlowrFilterExpression } from '../flowr-search-filters'; import { evalFilter } from '../flowr-search-filters'; @@ -20,6 +12,7 @@ import { enrich } from './search-enrichers'; import type { Mapper, MapperArguments } from './search-mappers'; import { map } from './search-mappers'; import type { ElementOf } from 'ts-essentials'; +import type { FlowrAnalysisInput } from '../../project/flowr-analyzer'; /** @@ -108,23 +101,23 @@ type CascadeEmpty[], NewE Elements extends [] ? FlowrSearchElements : FlowrSearchElements; function getFirst[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE + data: FlowrAnalysisInput, elements: FSE ): CascadeEmpty { return elements.mutate(e => [getFirstByLocation(e)] as Elements) as unknown as CascadeEmpty; } function getLast[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE): CascadeEmpty]> { + data: FlowrAnalysisInput, elements: FSE): CascadeEmpty]> { return elements.mutate(e => [getLastByLocation(e)] as Elements) as unknown as CascadeEmpty]>; } function getIndex[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE, { index }: { index: number }): CascadeEmpty { + data: FlowrAnalysisInput, elements: FSE, { index }: { index: number }): CascadeEmpty { return elements.mutate(e => [sortFully(e)[index]] as Elements) as unknown as CascadeEmpty; } function getSelect[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE, { select }: { select: number[] }): CascadeEmpty { + data: FlowrAnalysisInput, elements: FSE, { select }: { select: number[] }): CascadeEmpty { return elements.mutate(e => { sortFully(e); return select.map(i => e[i]).filter(isNotUndefined) as Elements; @@ -132,7 +125,7 @@ function getSelect[], FSE } function getTail[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE): CascadeEmpty> { + data: FlowrAnalysisInput, elements: FSE): CascadeEmpty> { return elements.mutate(e => { const first = getFirstByLocation(e); return e.filter(el => el !== first) as Elements; @@ -140,45 +133,60 @@ function getTail[], FSE e } function getTake[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE, { count }: { count: number }): CascadeEmpty> { + data: FlowrAnalysisInput, elements: FSE, { count }: { count: number }): CascadeEmpty> { return elements.mutate(e => sortFully(e).slice(0, count) as Elements) as unknown as CascadeEmpty>; } function getSkip[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE, { count }: { count: number }): CascadeEmpty> { + data: FlowrAnalysisInput, elements: FSE, { count }: { count: number }): CascadeEmpty> { return elements.mutate(e => sortFully(e).slice(count) as Elements) as unknown as CascadeEmpty>; } -function getFilter[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE, { filter }: { filter: FlowrFilterExpression }): CascadeEmpty { +async function getFilter[], FSE extends FlowrSearchElements>( + data: FlowrAnalysisInput, elements: FSE, { filter }: { + filter: FlowrFilterExpression + }): Promise> { + const filterInput = { + normalize: await data.normalizedAst(), + dataflow: await data.dataflow(), + config: data.flowrConfig, + }; return elements.mutate( - e => e.filter(({ node }) => evalFilter(filter, { node, normalize: data.normalize, dataflow: data.dataflow })) as Elements + e => e.filter(({ node }) => evalFilter(filter, { ...filterInput, node })) as Elements ) as unknown as CascadeEmpty; } -function getWith[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE, { info, args }: { info: Enrichment, args?: EnrichmentArguments }): FlowrSearchElements[]> { +async function getWith[], FSE extends FlowrSearchElements>( + data: FlowrAnalysisInput, elements: FSE, { info, args }: { + info: Enrichment, + args?: EnrichmentArguments + }): Promise[]>> { + const enrichmentInput = { + normalize: await data.normalizedAst(), + dataflow: await data.dataflow(), + config: data.flowrConfig, + }; return elements.mutate( - elements => elements.map(e => enrich(e, data, info, args)) as (Elements & EnrichedFlowrSearchElement[]) + elements => elements.map(e => enrich(e, enrichmentInput, info, args)) as (Elements & EnrichedFlowrSearchElement[]) ) as unknown as FlowrSearchElements[]>; } function getMap[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE, { mapper, args }: { mapper: Mapper, args: MapperArguments }): FlowrSearchElements { + data: FlowrAnalysisInput, elements: FSE, { mapper, args }: { mapper: Mapper, args: MapperArguments }): FlowrSearchElements { return elements.mutate( elements => elements.flatMap(e => map(e, data, mapper, args)) as Elements ) as unknown as FlowrSearchElements; } -function getMerge[], FSE extends FlowrSearchElements>( +async function getMerge[], FSE extends FlowrSearchElements>( /* search has to be unknown because it is a recursive type */ - data: FlowrSearchInput, elements: FSE, other: { search: unknown[], generator: FlowrSearchGeneratorNode }): FlowrSearchElements[]> { - const resultOther = runSearch(other as FlowrSearch, data); + data: FlowrAnalysisInput, elements: FSE, other: { search: unknown[], generator: FlowrSearchGeneratorNode }): Promise[]>> { + const resultOther = await runSearch(other as FlowrSearch, data); return elements.addAll(resultOther); } function getUnique[], FSE extends FlowrSearchElements>( - data: FlowrSearchInput, elements: FSE): CascadeEmpty { + _data: FlowrAnalysisInput, elements: FSE): CascadeEmpty { return elements.mutate(e => e.reduce((acc, cur) => { if(!acc.some(el => el.node.id === cur.node.id)) { diff --git a/test/functionality/_helper/linter.ts b/test/functionality/_helper/linter.ts index dad80d3f9fc..7c23316d137 100644 --- a/test/functionality/_helper/linter.ts +++ b/test/functionality/_helper/linter.ts @@ -8,29 +8,19 @@ import { LintingRules } from '../../../src/linter/linter-rules'; import type { TestLabel } from './label'; import { decorateLabelContext } from './label'; import { assert, test } from 'vitest'; -import { createDataflowPipeline } from '../../../src/core/steps/pipeline/default-pipelines'; import { requestFromInput } from '../../../src/r-bridge/retriever'; -import type { - NormalizedAst -} from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; -import { - deterministicCountingIdGenerator -} from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { NormalizedAst } from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; +import { deterministicCountingIdGenerator } from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; import { executeLintingRule } from '../../../src/linter/linter-executor'; import type { LintingRule } from '../../../src/linter/linter-format'; import { log } from '../../../src/util/log'; import type { DeepPartial } from 'ts-essentials'; import type { KnownParser } from '../../../src/r-bridge/parser'; -import type { - FlowrLaxSourcingOptions -} from '../../../src/config'; -import { - amendConfig, - defaultConfigOptions, - DropPathsOption -} from '../../../src/config'; +import type { FlowrLaxSourcingOptions } from '../../../src/config'; +import { amendConfig, defaultConfigOptions, DropPathsOption } from '../../../src/config'; import type { DataflowInformation } from '../../../src/dataflow/info'; import { graphToMermaidUrl } from '../../../src/util/mermaid/dfg'; +import { FlowrAnalyzerBuilder } from '../../../src/project/flowr-analyzer-builder'; export function assertLinter( name: string | TestLabel, @@ -49,14 +39,18 @@ export function assertLinter( }; return c; }); - const pipelineResults = await createDataflowPipeline(parser, { - request: requestFromInput(code), - getId: deterministicCountingIdGenerator(0), - overwriteFilePath: lintingRuleConfig?.useAsFilePath - }, flowrConfig).allRemainingSteps(); + + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) + .setInput({ + getId: deterministicCountingIdGenerator(0), + overwriteFilePath: lintingRuleConfig?.useAsFilePath + }) + .setParser(parser) + .setConfig(flowrConfig) + .build(); const rule = LintingRules[ruleName] as unknown as LintingRule, LintingRuleMetadata, LintingRuleConfig>; - const results = executeLintingRule(ruleName, { ...pipelineResults, config: flowrConfig }, lintingRuleConfig); + const results = await executeLintingRule(ruleName, analyzer, lintingRuleConfig); for(const [type, printer] of Object.entries({ text: (result: LintingRuleResult, metadata: LintingRuleMetadata) => `${rule.prettyPrint(result, metadata)} (${result.certainty})`, @@ -66,13 +60,13 @@ export function assertLinter( } if(typeof expected === 'function') { - expected = expected(pipelineResults.dataflow, pipelineResults.normalize); + expected = expected(await analyzer.dataflow(), await analyzer.normalizedAst()); } try { assert.deepEqual(results.results, expected, `Expected ${ruleName} to return ${JSON.stringify(expected)}, but got ${JSON.stringify(results)}`); } catch(e) { - console.error('dfg:', graphToMermaidUrl(pipelineResults.dataflow.graph)); + console.error('dfg:', graphToMermaidUrl((await analyzer.dataflow()).graph)); throw e; } if(expectedMetadata !== undefined) { diff --git a/test/functionality/_helper/query.ts b/test/functionality/_helper/query.ts index 9eb6626f98a..201b161cf47 100644 --- a/test/functionality/_helper/query.ts +++ b/test/functionality/_helper/query.ts @@ -1,7 +1,5 @@ import type { DEFAULT_DATAFLOW_PIPELINE } from '../../../src/core/steps/pipeline/default-pipelines'; -import { createDataflowPipeline } from '../../../src/core/steps/pipeline/default-pipelines'; import { requestFromInput } from '../../../src/r-bridge/retriever'; -import { deterministicCountingIdGenerator } from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; import type { Query, QueryResults, QueryResultsWithoutMeta } from '../../../src/queries/query'; import { executeQueries, SupportedQueries } from '../../../src/queries/query'; import type { VirtualQueryArgumentsWithType } from '../../../src/queries/virtual-query/virtual-queries'; @@ -17,6 +15,7 @@ import { defaultConfigOptions } from '../../../src/config'; import type { KnownParser } from '../../../src/r-bridge/parser'; import { extractCfg } from '../../../src/control-flow/extract-cfg'; import { getDummyFlowrProject } from '../../../src/project/flowr-project'; +import { FlowrAnalyzerBuilder } from '../../../src/project/flowr-analyzer-builder'; function normalizeResults(result: QueryResults): QueryResultsWithoutMeta { @@ -68,14 +67,13 @@ export function assertQuery< } } - const info = await createDataflowPipeline(parser, { - request: requestFromInput(code), - getId: deterministicCountingIdGenerator(0) - }, defaultConfigOptions).allRemainingSteps(); + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) + .setParser(parser) + .build(); const dummyProject = await getDummyFlowrProject(); - const result = executeQueries({ dataflow: info.dataflow, ast: info.normalize, config: defaultConfigOptions, libraries: dummyProject.libraries }, queries); + const result = await executeQueries({ input: analyzer, libraries: dummyProject.libraries }, queries); log.info(`total query time: ${result['.meta'].timing.toFixed(0)}ms (~1ms accuracy)`); @@ -84,11 +82,17 @@ export function assertQuery< /* expect them to be deeply equal */ try { // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const expectedNormalized = normalizeResults(typeof expected === 'function' ? await expected(info) : expected); + const expectedNormalized = normalizeResults(typeof expected === 'function' ? await expected( + { + parse: await analyzer.parseOutput(), + normalize: await analyzer.normalizedAst(), + dataflow: await analyzer.dataflow() + } + ) : expected); assert.deepStrictEqual(normalized, expectedNormalized, 'The result of the query does not match the expected result'); } /* v8 ignore next 3 */ catch(e: unknown) { - console.error('Dataflow-Graph', dataflowGraphToMermaidUrl(info.dataflow)); - console.error('Control-Flow-Graph', cfgToMermaidUrl(extractCfg(info.normalize, defaultConfigOptions, info.dataflow.graph), info.normalize)); + console.error('Dataflow-Graph', dataflowGraphToMermaidUrl(await analyzer.dataflow())); + console.error('Control-Flow-Graph', cfgToMermaidUrl(extractCfg(await analyzer.normalizedAst(), defaultConfigOptions, (await analyzer.dataflow()).graph), await analyzer.normalizedAst())); throw e; } }); diff --git a/test/functionality/_helper/search.ts b/test/functionality/_helper/search.ts index 1084e31b403..721a32979e9 100644 --- a/test/functionality/_helper/search.ts +++ b/test/functionality/_helper/search.ts @@ -1,11 +1,14 @@ import type { TestLabel } from './label'; import { decorateLabelContext } from './label'; -import type { DEFAULT_DATAFLOW_PIPELINE } from '../../../src/core/steps/pipeline/default-pipelines'; -import { createDataflowPipeline } from '../../../src/core/steps/pipeline/default-pipelines'; import { assert, beforeAll, describe, test } from 'vitest'; import { requestFromInput } from '../../../src/r-bridge/retriever'; -import type { ParentInformation } from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; -import { deterministicCountingIdGenerator } from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { + NormalizedAst, + ParentInformation +} from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; +import { + deterministicCountingIdGenerator +} from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; import { dataflowGraphToMermaidUrl } from '../../../src/core/print/dataflow-printer'; import type { FlowrSearchLike } from '../../../src/search/flowr-search-builder'; import { getFlowrSearch } from '../../../src/search/flowr-search-builder'; @@ -13,14 +16,15 @@ import type { NodeId } from '../../../src/r-bridge/lang-4.x/ast/model/processing import { runSearch } from '../../../src/search/flowr-search-executor'; import { arrayEqual } from '../../../src/util/collections/arrays'; import { type SingleSlicingCriterion, slicingCriterionToId } from '../../../src/slicing/criterion/parse'; -import type { PipelineOutput } from '../../../src/core/steps/pipeline/pipeline'; import { guard, isNotUndefined } from '../../../src/util/assert'; import { flowrSearchToAscii } from '../../../src/search/flowr-search-printer'; -import { defaultConfigOptions } from '../../../src/config'; import type { FlowrSearchElement } from '../../../src/search/flowr-search'; import type { Enrichment, EnrichmentContent } from '../../../src/search/search-executor/search-enrichers'; import { enrichmentContent } from '../../../src/search/search-executor/search-enrichers'; import type { KnownParser } from '../../../src/r-bridge/parser'; +import { FlowrAnalyzerBuilder } from '../../../src/project/flowr-analyzer-builder'; +import type { FlowrAnalyzer } from '../../../src/project/flowr-analyzer'; +import type { DataflowInformation } from '../../../src/dataflow/info'; /** * Asserts the result of a search or a set of searches (all of which should return the same result)! @@ -35,27 +39,36 @@ export function assertSearch( ) { const effectiveName = decorateLabelContext(name, ['search']); describe(effectiveName, () => { - let results: PipelineOutput | undefined; + let analyzer: FlowrAnalyzer | undefined; + let dataflow: DataflowInformation | undefined; + let ast: NormalizedAst | undefined; + beforeAll(async() => { - results = await createDataflowPipeline(parser, { - request: requestFromInput(code), - getId: deterministicCountingIdGenerator(0) - }, defaultConfigOptions).allRemainingSteps(); + analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) + .setInput({ + getId: deterministicCountingIdGenerator(0) + }) + .setParser(parser) + .build(); + dataflow = await analyzer.dataflow(); + ast = await analyzer.normalizedAst(); }); describe.each([true, false])('optimize %s', optimize => { - test.each(searches)('%s', search => { - guard(isNotUndefined(results), 'Results must be defined'); - const info = results; + test.each(searches)('%s', async search => { + guard(isNotUndefined(analyzer), 'Analyzer must be defined'); + guard(isNotUndefined(dataflow), 'Dataflow must be defined'); + guard(isNotUndefined(ast), 'Normalized AST must be defined'); search = getFlowrSearch(search, optimize); - const result = runSearch(search, { ...info, config: defaultConfigOptions }); + const result = await runSearch(search, analyzer); try { if(Array.isArray(expected)) { expected = expected.map(id => { try { - return slicingCriterionToId(id as SingleSlicingCriterion, info.normalize.idMap); + guard(isNotUndefined(ast), 'Normalized AST must be defined'); + return slicingCriterionToId(id as SingleSlicingCriterion, ast.idMap); } catch{ /* just keep it :D */ return id as NodeId; @@ -66,10 +79,13 @@ export function assertSearch( `Expected search results to match. Wanted: [${expected.join(', ')}], got: [${result.map(r => r.node.info.id).join(', ')}]`); } else { const expectedFunc = expected as (result: FlowrSearchElement[]) => boolean; - assert(expectedFunc(result), `Expected search results ${JSON.stringify(results)} to match expected function`); + assert(expectedFunc(result), `Expected search results ${JSON.stringify({ + normalize: ast, + dataflow: dataflow + })} to match expected function`); } } /* v8 ignore next 4 */ catch(e: unknown) { - console.error('Dataflow-Graph', dataflowGraphToMermaidUrl(info.dataflow)); + console.error('Dataflow-Graph', dataflowGraphToMermaidUrl(dataflow)); console.error('Search', flowrSearchToAscii(search)); throw e; } diff --git a/test/functionality/_helper/shell.ts b/test/functionality/_helper/shell.ts index 28ddfe16574..8c4d724b680 100644 --- a/test/functionality/_helper/shell.ts +++ b/test/functionality/_helper/shell.ts @@ -18,7 +18,6 @@ import type { } from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; import { deterministicCountingIdGenerator } from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; import { - DEFAULT_DATAFLOW_PIPELINE, DEFAULT_NORMALIZE_PIPELINE, DEFAULT_SLICE_AND_RECONSTRUCT_PIPELINE, TREE_SITTER_NORMALIZE_PIPELINE, @@ -36,11 +35,10 @@ import { resolveDataflowGraph } from '../../../src/dataflow/graph/resolve-graph' import { afterAll, assert, beforeAll, describe, test } from 'vitest'; import semver from 'semver/preload'; import { TreeSitterExecutor } from '../../../src/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; -import type { Pipeline, PipelineOutput } from '../../../src/core/steps/pipeline/pipeline'; +import type { PipelineOutput } from '../../../src/core/steps/pipeline/pipeline'; import type { FlowrSearchLike } from '../../../src/search/flowr-search-builder'; import { runSearch } from '../../../src/search/flowr-search-executor'; import type { ContainerIndex } from '../../../src/dataflow/graph/vertex'; -import type { DataflowInformation } from '../../../src/dataflow/info'; import type { REnvironmentInformation } from '../../../src/dataflow/environments/environment'; import { resolveByName } from '../../../src/dataflow/environments/resolve-by-name'; import type { GraphDifferenceReport, ProblematicDiffInfo } from '../../../src/util/diff-graph'; @@ -50,6 +48,8 @@ import type { CfgProperty } from '../../../src/control-flow/cfg-properties'; import { assertCfgSatisfiesProperties } from '../../../src/control-flow/cfg-properties'; import type { FlowrConfigOptions } from '../../../src/config'; import { cloneConfig, defaultConfigOptions } from '../../../src/config'; +import { FlowrAnalyzerBuilder } from '../../../src/project/flowr-analyzer-builder'; +import type { FlowrAnalysisInput } from '../../../src/project/flowr-analyzer'; export const testWithShell = (msg: string, fn: (shell: RShell, test: unknown) => void | Promise) => { return test(msg, async function(this: unknown): Promise { @@ -348,29 +348,34 @@ function cropIfTooLong(str: string): string { * Especially the `resolveIdsAsCriterion` and the `expectIsSubgraph` are interesting as they allow you for rather * flexible matching of the expected graph. */ -export function assertDataflow

( +export function assertDataflow( name: string | TestLabel, shell: RShell, input: string | RParseRequests, - expected: DataflowGraph | ((data: PipelineOutput

& { normalize: NormalizedAst, dataflow: DataflowInformation }) => DataflowGraph), + expected: DataflowGraph | ((input: FlowrAnalysisInput) => Promise), userConfig?: Partial, startIndexForDeterministicIds = 0, config = cloneConfig(defaultConfigOptions) ): void { const effectiveName = decorateLabelContext(name, ['dataflow']); test.skipIf(skipTestBecauseConfigNotMet(userConfig))(`${effectiveName} (input: ${cropIfTooLong(JSON.stringify(input))})`, async function() { - const info = await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, { - parser: shell, - request: typeof input === 'string' ? requestFromInput(input) : input, - getId: deterministicCountingIdGenerator(startIndexForDeterministicIds) - }, config).allRemainingSteps(); + const analyzer = await new FlowrAnalyzerBuilder(typeof input === 'string' ? requestFromInput(input) : input) + .setInput({ + getId: deterministicCountingIdGenerator(startIndexForDeterministicIds) + }) + .setConfig(config) + .setParser(shell) + .build(); if(typeof expected === 'function') { - expected = expected(info); + expected = await expected(analyzer); } + const normalize = await analyzer.normalizedAst(); + const dataflow = await analyzer.dataflow(); + // assign the same id map to the expected graph, so that resolves work as expected - expected.setIdMap(info.normalize.idMap); + expected.setIdMap(normalize.idMap); if(userConfig?.resolveIdsAsCriterion) { expected = resolveDataflowGraph(expected); @@ -378,7 +383,7 @@ export function assertDataflow

( const report: GraphDifferenceReport = diffOfDataflowGraphs( { name: 'expected', graph: expected }, - { name: 'got', graph: info.dataflow.graph }, + { name: 'got', graph: dataflow.graph }, { leftIsSubgraph: userConfig?.expectIsSubgraph } @@ -389,13 +394,13 @@ export function assertDataflow

( } /* v8 ignore start */ catch(e) { const diff = diffGraphsToMermaidUrl( { label: 'expected', graph: expected, mark: mapProblematicNodesToIds(report.problematic()) }, - { label: 'got', graph: info.dataflow.graph, mark: mapProblematicNodesToIds(report.problematic()) }, + { label: 'got', graph: dataflow.graph, mark: mapProblematicNodesToIds(report.problematic()) }, `%% ${JSON.stringify(input).replace(/\n/g, '\n%% ')}\n` + report.comments()?.map(n => `%% ${n}\n`).join('') + '\n' ); - console.error('ast', normalizedAstToMermaidUrl(info.normalize.ast)); + console.error('ast', normalizedAstToMermaidUrl(normalize.ast)); - console.error('best-effort reconstruction:\n', printAsBuilder(info.dataflow.graph)); + console.error('best-effort reconstruction:\n', printAsBuilder(dataflow.graph)); console.error('diff:\n', diff); throw e; @@ -583,18 +588,20 @@ export function assertContainerIndicesDefinition( ) { const effectiveName = decorateLabelContext(name, ['dataflow']); test.skipIf(skipTestBecauseConfigNotMet(userConfig))(`${effectiveName} (input: ${cropIfTooLong(JSON.stringify(input))})`, async function() { - const analysis = await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, { - parser: shell, - request: requestFromInput(input), - }, config).allRemainingSteps(); - const result = runSearch(search, { ...analysis, config: defaultConfigOptions }); + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(input)) + .setConfig(config) + .setParser(shell) + .build(); + const dataflow = await analyzer.dataflow(); + const normalize = await analyzer.normalizedAst(); + const result = await runSearch(search, analyzer); let findIndices: (id: NodeId) => ContainerIndex[] | undefined; if(userConfig.searchIn === 'dfg') { - findIndices = id => findInDfg(id, analysis.dataflow.graph); + findIndices = id => findInDfg(id, dataflow.graph); } else if(userConfig.searchIn === 'env') { - findIndices = id => findInEnv(id, analysis.normalize, analysis.dataflow.graph, analysis.dataflow.environment); + findIndices = id => findInEnv(id, normalize, dataflow.graph, dataflow.environment); } else { - findIndices = id => findInDfg(id, analysis.dataflow.graph) ?? findInEnv(id, analysis.normalize, analysis.dataflow.graph, analysis.dataflow.environment); + findIndices = id => findInDfg(id, dataflow.graph) ?? findInEnv(id, normalize, dataflow.graph, dataflow.environment); } @@ -616,11 +623,11 @@ export function assertContainerIndicesDefinition( try { assert.strictEqual( actual, expected, - `got: ${actual}, vs. expected: ${expected}, for input ${input}, url: ${graphToMermaidUrl(analysis.dataflow.graph, true)}` + `got: ${actual}, vs. expected: ${expected}, for input ${input}, url: ${graphToMermaidUrl(dataflow.graph, true)}` ); } /* v8 ignore start */ catch(e) { console.error(`got:\n${actual}\nvs. expected:\n${expected}`); - console.error(normalizedAstToMermaidUrl(analysis.normalize.ast)); + console.error(normalizedAstToMermaidUrl(normalize.ast)); throw e; } /* v8 ignore stop */ } diff --git a/test/functionality/dataflow/pointer-analysis/container-single-index-based-access.test.ts b/test/functionality/dataflow/pointer-analysis/container-single-index-based-access.test.ts index db1d8bf6772..3cf8b7b7489 100644 --- a/test/functionality/dataflow/pointer-analysis/container-single-index-based-access.test.ts +++ b/test/functionality/dataflow/pointer-analysis/container-single-index-based-access.test.ts @@ -42,8 +42,8 @@ describe.sequential('Container Single Index Based Access', withShell(shell => { shell, `numbers <- ${def('1', '2')} ${acc('numbers', 2)}`, - (data) => emptyGraph() - .readsQuery(queryAccInLine(2), queryArg(2, '2', 1), { ...data, config: defaultConfigOptions }), + async(data) => await emptyGraph() + .readsQuery(queryAccInLine(2), queryArg(2, '2', 1), data), { expectIsSubgraph: true, resolveIdsAsCriterion: true, @@ -58,8 +58,8 @@ describe.sequential('Container Single Index Based Access', withShell(shell => { shell, `numbers <- ${def('1', 'c(2, 3)', '4')} ${acc('numbers', 2)}`, - (data) => emptyGraph() - .readsQuery(queryAccInLine(2), queryUnnamedArg('2', 1), { ...data, config: defaultConfigOptions }), + async(data) => await emptyGraph() + .readsQuery(queryAccInLine(2), queryUnnamedArg('2', 1), data), { expectIsSubgraph: true, resolveIdsAsCriterion: true, @@ -74,7 +74,7 @@ describe.sequential('Container Single Index Based Access', withShell(shell => { `numbers <- ${def('1', 'list(a = 2, b = 3)', '4')} ${acc('numbers', 2)}`, (data) => emptyGraph() - .readsQuery(queryAccInLine(2), queryNamedArg('a', 1), { ...data, config: defaultConfigOptions }), + .readsQuery(queryAccInLine(2), queryNamedArg('a', 1), data), { expectIsSubgraph: true, resolveIdsAsCriterion: true, @@ -88,8 +88,8 @@ describe.sequential('Container Single Index Based Access', withShell(shell => { shell, `numbers <- ${def('1', 'list(2, 3)', '4')} ${acc('numbers', 2)}`, - (data) => emptyGraph() - .readsQuery(queryAccInLine(2), queryUnnamedArg('2', 1), { ...data, config: defaultConfigOptions }), + async(data) => emptyGraph() + .readsQuery(queryAccInLine(2), queryUnnamedArg('2', 1), data), { expectIsSubgraph: true, resolveIdsAsCriterion: true, @@ -105,8 +105,8 @@ describe.sequential('Container Single Index Based Access', withShell(shell => { shell, `numbers <- ${def('1', ['2', '3'], '4')} ${acc('numbers', 2, 1)}`, - (data) => emptyGraph() - .readsQuery({ query: Q.varInLine(type, 2).last() }, queryArg(3, '2', 1), { ...data, config: defaultConfigOptions }), + async(data) => emptyGraph() + .readsQuery({ query: Q.varInLine(type, 2).last() }, queryArg(3, '2', 1), data), { expectIsSubgraph: true, resolveIdsAsCriterion: true, @@ -124,7 +124,7 @@ describe.sequential('Container Single Index Based Access', withShell(shell => { c <- ${def('b')} ${acc(acc(acc('c', 1), 42), 1)}`, (data) => emptyGraph() - .readsQuery(queryAccInLine(4, (query => query.last())), queryArg(1, 'b', 3), { ...data, config: defaultConfigOptions }), + .readsQuery(queryAccInLine(4, (query => query.last())), queryArg(1, 'b', 3), data), { expectIsSubgraph: true, resolveIdsAsCriterion: true, @@ -144,8 +144,8 @@ describe.sequential('Container Single Index Based Access', withShell(shell => { ${acc('numbers', 1)} <- 5 ${acc('numbers', 1)}`, (data) => emptyGraph() - .readsQuery(queryAccInLine(3), queryArg(1, '1', 1), { ...data, config: defaultConfigOptions }) - .readsQuery(queryAccInLine(3), queryAccInLine(2), { ...data, config: defaultConfigOptions }), + .readsQuery(queryAccInLine(3), queryArg(1, '1', 1), data) + .then((builder) => builder.readsQuery(queryAccInLine(3), queryAccInLine(2), data)), { expectIsSubgraph: true, resolveIdsAsCriterion: true, @@ -164,8 +164,8 @@ describe.sequential('Container Single Index Based Access', withShell(shell => { ${acc('numbers', 4)} <- 1 ${acc('numbers', 1)}`, (data) => emptyGraph() - .readsQuery(queryAccInLine(6), queryArg(1, '1', 1), { ...data, config: defaultConfigOptions }) - .readsQuery(queryAccInLine(6), queryAccInLine(2), { ...data, config: defaultConfigOptions }), + .readsQuery(queryAccInLine(6), queryArg(1, '1', 1), data) + .then(builder => builder.readsQuery(queryAccInLine(6), queryAccInLine(2), data)), // not reads other indices { expectIsSubgraph: true, @@ -183,12 +183,13 @@ describe.sequential('Container Single Index Based Access', withShell(shell => { ${acc('numbers', 1)} <- 1 print(${acc('numbers', 1)})`, (data) => emptyGraph() - .readsQuery({ query: Q.varInLine('numbers', 2).last() }, { target: '1@numbers' }, { ...data, config: defaultConfigOptions }) - .definedByQuery( - { query: Q.varInLine('numbers', 2).first() }, - { query: Q.varInLine('numbers', 2).last() }, - { ...data, config: defaultConfigOptions }, - ), + .readsQuery({ query: Q.varInLine('numbers', 2).last() }, { target: '1@numbers' }, data) + .then(builder => + builder.definedByQuery( + { query: Q.varInLine('numbers', 2).first() }, + { query: Q.varInLine('numbers', 2).last() }, + data, + )), { expectIsSubgraph: true, resolveIdsAsCriterion: true, @@ -208,10 +209,10 @@ ${acc('numbers', 1)} <- 1 ${acc('numbers', 2)} <- 2 ${accS('numbers', 'foo()')}`, (data) => emptyGraph() - .readsQuery(queryAccInLine(4), queryArg(1, '1', 1), { ...data, config: defaultConfigOptions }) - .readsQuery(queryAccInLine(4), queryArg(2, '2', 1), { ...data, config: defaultConfigOptions }) - .readsQuery(queryAccInLine(4), queryAccInLine(2), { ...data, config: defaultConfigOptions }) - .readsQuery(queryAccInLine(4), queryAccInLine(3), { ...data, config: defaultConfigOptions }) + .readsQuery(queryAccInLine(4), queryArg(1, '1', 1), data) + .then(builder => builder.readsQuery(queryAccInLine(4), queryArg(2, '2', 1), data)) + .then(builder => builder.readsQuery(queryAccInLine(4), queryAccInLine(2), data)) + .then(builder => builder.readsQuery(queryAccInLine(4), queryAccInLine(3), data)) , { expectIsSubgraph: true, diff --git a/test/functionality/slicing/slicing.bench.ts b/test/functionality/slicing/slicing.bench.ts index 435cdf664bd..2f7b77c94a2 100644 --- a/test/functionality/slicing/slicing.bench.ts +++ b/test/functionality/slicing/slicing.bench.ts @@ -1,7 +1,4 @@ import { bench, describe } from 'vitest'; -import type { PipelineOutput } from '../../../src/core/steps/pipeline/pipeline'; -import type { TREE_SITTER_DATAFLOW_PIPELINE } from '../../../src/core/steps/pipeline/default-pipelines'; -import { createDataflowPipeline } from '../../../src/core/steps/pipeline/default-pipelines'; import { TreeSitterExecutor } from '../../../src/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; import { requestFromInput } from '../../../src/r-bridge/retriever'; import type { NodeId } from '../../../src/r-bridge/lang-4.x/ast/model/processing/node-id'; @@ -9,21 +6,20 @@ import { runSearch } from '../../../src/search/flowr-search-executor'; import { Q } from '../../../src/search/flowr-search-builder'; import { staticSlicing } from '../../../src/slicing/static/static-slicer'; import { guard } from '../../../src/util/assert'; -import { defaultConfigOptions } from '../../../src/config'; +import { FlowrAnalyzerBuilder } from '../../../src/project/flowr-analyzer-builder'; +import type { DataflowInformation } from '../../../src/dataflow/info'; +import type { NormalizedAst } from '../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; describe('slicing', () => { - let result: PipelineOutput | undefined = undefined; + let result: { dataflow: DataflowInformation; normalize: NormalizedAst } | undefined = undefined; let ids: NodeId[] | undefined = undefined; for(const threshold of [1, 10, 100, 200]) { bench(`slice (threshold: ${threshold})`, async() => { if(!result) { - await TreeSitterExecutor.initTreeSitter(); - const exec = new TreeSitterExecutor(); - result = await createDataflowPipeline(exec, { - /* make it hurt! */ - request: requestFromInput(` + /* make it hurt! */ + const request = requestFromInput(` for(i in 1:5) { if(u) { x <- c(1, 2, 3) @@ -38,9 +34,13 @@ for(i in 1:5) { x[1] <- 4 x[2] <- x[1] + x[3] } - `.trim().repeat(200) + '\nprint(x + f(1, function(i) x[[i]] + 2, 3))'), - }, defaultConfigOptions).allRemainingSteps(); - ids = runSearch(Q.var('print').first(), { ...result, config: defaultConfigOptions }).map(n => n.node.info.id); + `.trim().repeat(200) + '\nprint(x + f(1, function(i) x[[i]] + 2, 3))'); + // TODO TSchoeller Use executor + await TreeSitterExecutor.initTreeSitter(); + const exec = new TreeSitterExecutor(); + const analyzer = await new FlowrAnalyzerBuilder(request).build(); + result = { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalizedAst() }; + ids = (await runSearch(Q.var('print').first(), analyzer)).map(n => n.node.info.id); } guard(result !== undefined && ids !== undefined, () => 'no result'); staticSlicing(result.dataflow.graph, result.normalize, [`$${ids[0]}`], threshold); From bc4bc922e3b013ad5564ecf884daa4a282475613 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Mon, 1 Sep 2025 10:04:27 +0200 Subject: [PATCH 39/70] feat: use analyzer to get cfg in queries --- src/project/flowr-analyzer.ts | 1 + .../catalog/call-context-query/call-context-query-executor.ts | 3 +-- .../catalog/control-flow-query/control-flow-query-executor.ts | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index ea47bfdb078..616d408908f 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -19,6 +19,7 @@ import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize export type FlowrAnalysisInput = { normalizedAst(force?: boolean): Promise; dataflow(force?: boolean): Promise; + controlFlow(simplifications?: readonly CfgSimplificationPassName[], useDataflow?: boolean, force?: boolean): Promise; flowrConfig: FlowrConfigOptions; } diff --git a/src/queries/catalog/call-context-query/call-context-query-executor.ts b/src/queries/catalog/call-context-query/call-context-query-executor.ts index 626e5c2d66a..db05fdf7f52 100644 --- a/src/queries/catalog/call-context-query/call-context-query-executor.ts +++ b/src/queries/catalog/call-context-query/call-context-query-executor.ts @@ -11,7 +11,6 @@ import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/nod import { recoverContent } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id'; import { VertexType } from '../../../dataflow/graph/vertex'; import { edgeIncludesType, EdgeType } from '../../../dataflow/graph/edge'; -import { extractCfg } from '../../../control-flow/extract-cfg'; import { TwoLayerCollector } from '../../two-layer-collector'; import { compactRecord } from '../../../util/objects'; @@ -214,7 +213,7 @@ export async function executeCallContextQueries({ input }: BasicQueryData, queri let cfg = undefined; if(requiresCfg) { - cfg = extractCfg(ast, input.flowrConfig, dataflow.graph, []); + cfg = await input.controlFlow([], true); } const queriesWhichWantAliases = promotedQueries.filter(q => q.includeAliases); diff --git a/src/queries/catalog/control-flow-query/control-flow-query-executor.ts b/src/queries/catalog/control-flow-query/control-flow-query-executor.ts index 82deb06a91b..8f321abf972 100644 --- a/src/queries/catalog/control-flow-query/control-flow-query-executor.ts +++ b/src/queries/catalog/control-flow-query/control-flow-query-executor.ts @@ -1,7 +1,6 @@ import { log } from '../../../util/log'; import type { ControlFlowQuery, ControlFlowQueryResult } from './control-flow-query-format'; import type { BasicQueryData } from '../../base-query-format'; -import { extractCfg } from '../../../control-flow/extract-cfg'; export async function executeControlFlowQuery({ input }: BasicQueryData, queries: readonly ControlFlowQuery[]): Promise { @@ -12,7 +11,7 @@ export async function executeControlFlowQuery({ input }: BasicQueryData, queries const query = queries[0]; const start = Date.now(); - const controlFlow = extractCfg(await input.normalizedAst(), input.flowrConfig, (await input.dataflow()).graph, query.config?.simplificationPasses); + const controlFlow = await input.controlFlow(query.config?.simplificationPasses, true); return { '.meta': { timing: Date.now() - start From f453309d40237be28cd82733711728272ef37bf4 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Mon, 1 Sep 2025 17:58:21 +0200 Subject: [PATCH 40/70] feat-fix: prevent excessive R-Shell creation --- src/project/flowr-analyzer-builder.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 3a7fc15f079..1ec2aabf484 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -56,9 +56,13 @@ export class FlowrAnalyzerBuilder { } public async build(): Promise { - const engines = await retrieveEngineInstances(this.flowrConfig); - // TODO TSchoeller Is this complexity necessary? - const parser = this.parser ?? engines.engines[engines.default] as KnownParser; + let parser: KnownParser; + if(this.parser) { + parser = this.parser; + } else { + const engines = await retrieveEngineInstances(this.flowrConfig); + parser = this.parser ?? engines.engines[engines.default] as KnownParser; + } return new FlowrAnalyzer( this.flowrConfig, From 296cfdd79fb7bfac3842cc70c03cffbae38add26 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Thu, 4 Sep 2025 09:42:43 +0200 Subject: [PATCH 41/70] feat-fix: set correct slicing params noMagicComments query parameter will be used to set autoSelectIf --- src/cli/repl/server/connection.ts | 5 +---- src/cli/repl/server/messages/message-slice.ts | 7 +++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index db0b339bf9c..fb944023e7c 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -256,14 +256,11 @@ export class FlowRServerConnection { } try { - // TODO TSchoeller Previously, this did update the slice request const result = await fileInformation.analyzer.query([{ type: 'static-slice', criteria: request.criterion, noMagicComments: request.noMagicComments, - // TODO TSchoeller autoSelectIf -> How to pass? - // direction: request.direction - //autoSelectIf: request.noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect) + direction: request.direction }]); sendMessage(this.socket, { diff --git a/src/cli/repl/server/messages/message-slice.ts b/src/cli/repl/server/messages/message-slice.ts index 3365dae7b9c..8190ab3bb14 100644 --- a/src/cli/repl/server/messages/message-slice.ts +++ b/src/cli/repl/server/messages/message-slice.ts @@ -2,7 +2,10 @@ import type { IdMessageBase, MessageDefinition } from './all-messages'; import * as Joi from 'joi'; import type { SlicingCriteria } from '../../../../slicing/criterion/parse'; import type { PipelineOutput } from '../../../../core/steps/pipeline/pipeline'; -import type { DEFAULT_DATAFLOW_PIPELINE, DEFAULT_SLICING_PIPELINE } from '../../../../core/steps/pipeline/default-pipelines'; +import type { + DEFAULT_DATAFLOW_PIPELINE, + DEFAULT_SLICING_PIPELINE +} from '../../../../core/steps/pipeline/default-pipelines'; import { SliceDirection } from '../../../../core/steps/all/static-slicing/00-slice'; /** @@ -19,7 +22,7 @@ export interface SliceRequestMessage extends IdMessageBase { /** The direction to slice in. Defaults to backward slicing if unset. */ direction?: SliceDirection, /** - * Should the magic comments (force-including lines within the slice) be ignord? + * Should the magic comments (force-including lines within the slice) be ignored? */ noMagicComments?: boolean } From 8298aff4a02ee99f746d66f166a6b656f31f0b52 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Thu, 4 Sep 2025 09:50:01 +0200 Subject: [PATCH 42/70] feat-fix: clean-up --- src/cli/repl/server/connection.ts | 1 - src/project/flowr-analyzer.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index fb944023e7c..6bd921c5aad 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -389,7 +389,6 @@ export class FlowRServerConnection { export function sanitizeAnalysisResults(parse: ParseStepOutput, normalize: NormalizedAst, dataflow: DataflowInformation): DeepPartial> { return { - // TODO Results missing? parse, normalize: { ...normalize, diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 3390981c62f..2e43546df9b 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -53,7 +53,6 @@ export class FlowrAnalyzer { } // TODO TSchoeller Fix type - // TODO TSchoeller Do we want to expose parsing as primary view in addition to the AST, CFG, and dataflow? public async parseOutput(force?: boolean): Promise & PipelinePerStepMetaInformation> { if(this.parse && !force) { return { From 4f89042f524c1e02957c384b3a82f9259b2013da Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Thu, 4 Sep 2025 17:57:33 +0200 Subject: [PATCH 43/70] feat-fix(analyzer): cfg caching --- src/control-flow/extract-cfg.ts | 2 +- src/project/flowr-analyzer.ts | 45 +++++++++++++------ .../search-executor/search-enrichers.ts | 4 +- src/util/collections/arraymap.ts | 25 +++++++++++ .../controlflow/assert-control-flow-graph.ts | 23 +++++----- 5 files changed, 71 insertions(+), 28 deletions(-) create mode 100644 src/util/collections/arraymap.ts diff --git a/src/control-flow/extract-cfg.ts b/src/control-flow/extract-cfg.ts index 06c56f4c78d..7060a06097f 100644 --- a/src/control-flow/extract-cfg.ts +++ b/src/control-flow/extract-cfg.ts @@ -93,7 +93,7 @@ export function extractCfg( } /** - * A version of {@link extractCfg} that is much quicker and does not apply any simplifciations or dataflow information. + * A version of {@link extractCfg} that is much quicker and does not apply any simplifications or dataflow information. */ export function extractCfgQuick(ast: NormalizedAst) { return foldAst(ast.ast, cfgFolds); diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 2e43546df9b..0ebd7d01a8e 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -15,6 +15,7 @@ import type { DataflowInformation } from '../dataflow/info'; import type { CfgSimplificationPassName } from '../control-flow/cfg-simplification'; import type { PipelinePerStepMetaInformation } from '../core/steps/pipeline/pipeline'; import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize'; +import { ArrayMap } from '../util/collections/arraymap'; export type FlowrAnalysisInput = { normalizedAst(force?: boolean): Promise; @@ -23,6 +24,11 @@ export type FlowrAnalysisInput = { flowrConfig: FlowrConfigOptions; } +interface ControlFlowCache { + simplified: ArrayMap, + quick: ControlFlowInformation +} + export class FlowrAnalyzer { public readonly flowrConfig: FlowrConfigOptions; private readonly request: RParseRequests; @@ -32,8 +38,10 @@ export class FlowrAnalyzer { private parse = undefined as unknown as ParseStepOutput; private ast = undefined as unknown as NormalizedAst; private dataflowInfo = undefined as unknown as DataflowInformation; - private controlFlowInfo = undefined as unknown as ControlFlowInformation; - private simpleControlFlowInfo = undefined as unknown as ControlFlowInformation; // TODO Differentiate between simple and regular CFG + private controlFlowInfos: ControlFlowCache = { + simplified: new ArrayMap(), + quick: undefined as unknown as ControlFlowInformation + }; constructor(config: FlowrConfigOptions, parser: KnownParser, request: RParseRequests, requiredInput: Omit) { this.flowrConfig = config; @@ -45,7 +53,10 @@ export class FlowrAnalyzer { public reset() { this.ast = undefined as unknown as NormalizedAst; this.dataflowInfo = undefined as unknown as DataflowInformation; - this.controlFlowInfo = undefined as unknown as ControlFlowInformation; + this.controlFlowInfos = { + simplified: new ArrayMap(), + quick: undefined as unknown as ControlFlowInformation + }; } public parserName(): string { @@ -109,8 +120,11 @@ export class FlowrAnalyzer { } public async controlFlow(simplifications?: readonly CfgSimplificationPassName[], useDataflow?: boolean, force?: boolean): Promise { - if(this.controlFlowInfo && !force) { - return this.controlFlowInfo; + if(!force) { + const value = this.controlFlowInfos.simplified.get(simplifications ?? []); + if(value !== undefined) { + return value; + } } if(force || !this.ast) { @@ -122,16 +136,21 @@ export class FlowrAnalyzer { } const result = extractCfg(this.ast, this.flowrConfig, this.dataflowInfo?.graph, simplifications); - this.controlFlowInfo = result; + this.controlFlowInfos.simplified.set(simplifications ?? [], result); return result; } - public async simpleControlFlow(simplifications?: readonly CfgSimplificationPassName[], force?: boolean): Promise { - if(this.simpleControlFlowInfo && !force) { - return this.simpleControlFlowInfo; - } else if(this.controlFlowInfo && !force) { - // Use the full CFG is it is already available - return this.controlFlowInfo; + public async controlFlowQuick(force?: boolean): Promise { + if(!force) { + if(this.controlFlowInfos.quick) { + return this.controlFlowInfos.quick; + } + + // Use the unsimplified CFG if it is already available + const value = this.controlFlowInfos.simplified.get([]); + if(value !== undefined) { + return value; + } } if(force || !this.ast) { @@ -139,7 +158,7 @@ export class FlowrAnalyzer { } const result = extractCfgQuick(this.ast); - this.simpleControlFlowInfo = result; + this.controlFlowInfos.quick = result; return result; } diff --git a/src/search/search-executor/search-enrichers.ts b/src/search/search-executor/search-enrichers.ts index 897520df8ff..4eea7fe04f5 100644 --- a/src/search/search-executor/search-enrichers.ts +++ b/src/search/search-executor/search-enrichers.ts @@ -14,7 +14,7 @@ import { identifyLinkToLastCallRelation } from '../../queries/catalog/call-context-query/identify-link-to-last-call-relation'; import { guard, isNotUndefined } from '../../util/assert'; -import { extractCfg, extractCfgQuick } from '../../control-flow/extract-cfg'; +import { extractCfgQuick } from '../../control-flow/extract-cfg'; import { getOriginInDfg, OriginType } from '../../dataflow/origin/dfg-get-origin'; import { type NodeId, recoverName } from '../../r-bridge/lang-4.x/ast/model/processing/node-id'; import type { ControlFlowInformation } from '../../control-flow/control-flow-graph'; @@ -201,7 +201,7 @@ export const Enrichments = { const content: CfgInformationSearchContent = { ...prev, - cfg: extractCfg(await data.normalizedAst(), data.flowrConfig, (await data.dataflow()).graph, args.simplificationPasses), + cfg: await data.controlFlow(args.simplificationPasses, true), }; if(args.checkReachable) { content.reachableNodes = cfgFindAllReachable(content.cfg); diff --git a/src/util/collections/arraymap.ts b/src/util/collections/arraymap.ts new file mode 100644 index 00000000000..68f44826e94 --- /dev/null +++ b/src/util/collections/arraymap.ts @@ -0,0 +1,25 @@ +/** + * A map type that uses an array of strings as key. + */ +export class ArrayMap { + private readonly internal = new Map(); + + private makeKey(key: readonly K[]): string { + return JSON.stringify(key); + } + + /** + * Sets a value for a given key. + */ + public set(key: readonly K[], v: V): void { + this.internal.set(this.makeKey(key), v); + } + + /** + * Return a value for the given key, if the key does not exist within the default map, + * this will invoke the generator and assign the produced value. + */ + public get(key: readonly K[]): V | undefined { + return this.internal.get(this.makeKey(key)); + } +} diff --git a/test/functionality/_helper/controlflow/assert-control-flow-graph.ts b/test/functionality/_helper/controlflow/assert-control-flow-graph.ts index d5ebf54af3b..3c84d0d633e 100644 --- a/test/functionality/_helper/controlflow/assert-control-flow-graph.ts +++ b/test/functionality/_helper/controlflow/assert-control-flow-graph.ts @@ -1,5 +1,4 @@ import { assert, test } from 'vitest'; -import { createDataflowPipeline } from '../../../../src/core/steps/pipeline/default-pipelines'; import { requestFromInput } from '../../../../src/r-bridge/retriever'; import { cfgToMermaidUrl } from '../../../../src/util/mermaid/cfg'; import type { KnownParser } from '../../../../src/r-bridge/parser'; @@ -9,14 +8,13 @@ import { diffOfControlFlowGraphs } from '../../../../src/control-flow/diff-cfg'; import type { GraphDifferenceReport } from '../../../../src/util/diff-graph'; import type { ControlFlowInformation } from '../../../../src/control-flow/control-flow-graph'; import { emptyControlFlowInformation } from '../../../../src/control-flow/control-flow-graph'; -import { extractCfg } from '../../../../src/control-flow/extract-cfg'; import type { CfgProperty } from '../../../../src/control-flow/cfg-properties'; import { assertCfgSatisfiesProperties } from '../../../../src/control-flow/cfg-properties'; import { cloneConfig, defaultConfigOptions } from '../../../../src/config'; import type { CfgSimplificationPassName } from '../../../../src/control-flow/cfg-simplification'; -import { simplifyControlFlowInformation } from '../../../../src/control-flow/cfg-simplification'; import type { DataflowInformation } from '../../../../src/dataflow/info'; import type { NormalizedAst } from '../../../../src/r-bridge/lang-4.x/ast/model/processing/decorate'; +import { FlowrAnalyzerBuilder } from '../../../../src/project/flowr-analyzer-builder'; function normAllIds(ids: readonly NodeId[]): NodeId[] { return ids.map(normalizeIdToNumberIfPossible); @@ -38,15 +36,16 @@ export function assertCfg(parser: KnownParser, code: string, partialExpected: Pa const expected: ControlFlowInformation = { ...emptyControlFlowInformation(), ...partialExpected }; return test(code, async()=> { const config = cloneConfig(defaultConfigOptions); - const result = await createDataflowPipeline(parser, { - request: requestFromInput(code) - }, config).allRemainingSteps(); - let cfg = extractCfg(result.normalize, config, result.dataflow?.graph); + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) + .setConfig(config) + .setParser(parser) + .build(); + let cfg = await analyzer.controlFlow(undefined, true); if(options?.withBasicBlocks) { - cfg = simplifyControlFlowInformation(cfg, { ast: result.normalize, dfg: result.dataflow.graph, config }, ['to-basic-blocks', 'remove-dead-code', ...options.simplificationPasses ?? []]); + cfg = await analyzer.controlFlow(['to-basic-blocks', 'remove-dead-code', ...options.simplificationPasses ?? []]); } else if(options?.simplificationPasses) { - cfg = simplifyControlFlowInformation(cfg, { ast: result.normalize, dfg: result.dataflow.graph, config }, options.simplificationPasses); + cfg = await analyzer.controlFlow(options.simplificationPasses ?? []); } let diff: GraphDifferenceReport | undefined; @@ -65,14 +64,14 @@ export function assertCfg(parser: KnownParser, code: string, partialExpected: Pa }); assert.isTrue(diff.isEqual(), 'graphs differ:' + (diff?.comments() ?? []).join('\n')); if(options?.additionalAsserts) { - options.additionalAsserts(cfg, result.normalize, result.dataflow); + options.additionalAsserts(cfg, await analyzer.normalizedAst(), await analyzer.dataflow()); } } /* v8 ignore next 7 */ catch(e: unknown) { if(diff) { console.error(diff.comments()); } - console.error(`expected: ${cfgToMermaidUrl(expected, result.normalize)}`); - console.error(`actual: ${cfgToMermaidUrl(cfg, result.normalize)}`); + console.error(`expected: ${cfgToMermaidUrl(expected, await analyzer.normalizedAst())}`); + console.error(`actual: ${cfgToMermaidUrl(cfg, await analyzer.normalizedAst())}`); throw e; } }); From b75dd86aa30970141f4fb38008f4c9f84ae42fab Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 5 Sep 2025 11:16:25 +0200 Subject: [PATCH 44/70] doc(analyzer): clean-up --- src/documentation/doc-util/doc-search.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/documentation/doc-util/doc-search.ts b/src/documentation/doc-util/doc-search.ts index aec7d3909b9..52db701bf67 100644 --- a/src/documentation/doc-util/doc-search.ts +++ b/src/documentation/doc-util/doc-search.ts @@ -1,7 +1,5 @@ import type { RShell } from '../../r-bridge/shell'; import type { SupportedQueryTypes } from '../../queries/query'; -import { PipelineExecutor } from '../../core/pipeline-executor'; -import { DEFAULT_DATAFLOW_PIPELINE } from '../../core/steps/pipeline/default-pipelines'; import { requestFromInput } from '../../r-bridge/retriever'; import { getFilePathMd } from './doc-files'; import type { SupportedVirtualQueryTypes } from '../../queries/virtual-query/virtual-queries'; @@ -13,7 +11,6 @@ import { runSearch } from '../../search/flowr-search-executor'; import { flowrSearchToCode, flowrSearchToMermaid } from '../../search/flowr-search-printer'; import { recoverContent } from '../../r-bridge/lang-4.x/ast/model/processing/node-id'; import { formatRange } from '../../util/mermaid/dfg'; -import { defaultConfigOptions } from '../../config'; import { FlowrAnalyzerBuilder } from '../../project/flowr-analyzer-builder'; export interface ShowSearchOptions { @@ -23,13 +20,8 @@ export interface ShowSearchOptions { export async function showSearch(shell: RShell, code: string, search: FlowrSearchLike, { collapseResult = true }: ShowSearchOptions = {}): Promise { const now = performance.now(); - const analysis = await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, { - parser: shell, - request: requestFromInput(code) - }, defaultConfigOptions).allRemainingSteps(); const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) .setParser(shell) - .setConfig(defaultConfigOptions) .build(); const result = await runSearch(search, analyzer); const duration = performance.now() - now; @@ -38,6 +30,8 @@ export async function showSearch(shell: RShell, code: string, search: FlowrSearc The search required _${printAsMs(duration)}_ (including parsing and normalization and the query) within the generation environment. `.trim(); + const dataflow = await analyzer.dataflow(); + return ` ${codeBlock('ts', flowrSearchToCode(search))} @@ -63,7 +57,7 @@ ${collapseResult ? '

Show Results `${node.info.id} ('${recoverContent(node.info.id, analysis.dataflow.graph)}') at L${formatRange(node.location)}`).join(', ') + result.getElements().map(({ node }) => `${node.info.id} ('${recoverContent(node.info.id, dataflow.graph)}') at L${formatRange(node.location)}`).join(', ') } ${metaInfo} From fef77d801ab5901483c7275695f4a04fff0dc4eb Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 5 Sep 2025 12:30:03 +0200 Subject: [PATCH 45/70] feat(config): allow config amendment without return --- src/config.ts | 7 +++++-- src/project/flowr-analyzer-builder.ts | 3 ++- test/functionality/_helper/linter.ts | 17 +++++++---------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/config.ts b/src/config.ts index b26cf0998eb..c95a3f0e76e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -334,8 +334,11 @@ export function parseConfig(jsonString: string): FlowrConfigOptions | undefined /** * Creates a new flowr config that has the updated values. */ -export function amendConfig(config: FlowrConfigOptions, amendmentFunc: (config: DeepWritable) => FlowrConfigOptions) { - return amendmentFunc(cloneConfig(config) as DeepWritable); +// eslint-disable-next-line @typescript-eslint/no-invalid-void-type +export function amendConfig(config: FlowrConfigOptions, amendmentFunc: (config: DeepWritable) => FlowrConfigOptions | void): FlowrConfigOptions { + const newConfig = cloneConfig(config); + amendmentFunc(newConfig as DeepWritable); + return newConfig; } export function cloneConfig(config: FlowrConfigOptions): FlowrConfigOptions { diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 1ec2aabf484..08c297bffc9 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -15,7 +15,8 @@ export class FlowrAnalyzerBuilder { private input?: Omit; private plugins: FlowrAnalyzerPlugin[]; - public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions): this { + // eslint-disable-next-line @typescript-eslint/no-invalid-void-type + public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions | void): this { this.flowrConfig = amendConfig(this.flowrConfig, func); return this; } diff --git a/test/functionality/_helper/linter.ts b/test/functionality/_helper/linter.ts index 122d6c82090..3821333b272 100644 --- a/test/functionality/_helper/linter.ts +++ b/test/functionality/_helper/linter.ts @@ -18,7 +18,7 @@ import { log } from '../../../src/util/log'; import type { DeepPartial } from 'ts-essentials'; import type { KnownParser } from '../../../src/r-bridge/parser'; import type { FlowrLaxSourcingOptions } from '../../../src/config'; -import { amendConfig, defaultConfigOptions, DropPathsOption } from '../../../src/config'; +import { DropPathsOption } from '../../../src/config'; import type { DataflowInformation } from '../../../src/dataflow/info'; import { graphToMermaidUrl } from '../../../src/util/mermaid/dfg'; import { FlowrAnalyzerBuilder } from '../../../src/project/flowr-analyzer-builder'; @@ -33,21 +33,18 @@ export function assertLinter( lintingRuleConfig?: DeepPartial> & { useAsFilePath?: string } ) { test(decorateLabelContext(name, ['linter']), async() => { - const flowrConfig = amendConfig(defaultConfigOptions, c => { - (c.solver.resolveSource as FlowrLaxSourcingOptions) = { - ...c.solver.resolveSource as FlowrLaxSourcingOptions, - dropPaths: DropPathsOption.All - }; - return c; - }); - const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) .setInput({ getId: deterministicCountingIdGenerator(0), overwriteFilePath: lintingRuleConfig?.useAsFilePath }) .setParser(parser) - .setConfig(flowrConfig) + .amendConfig(c => { + (c.solver.resolveSource as FlowrLaxSourcingOptions) = { + ...c.solver.resolveSource as FlowrLaxSourcingOptions, + dropPaths: DropPathsOption.All + }; + }) .build(); const rule = LintingRules[ruleName] as unknown as LintingRule, LintingRuleMetadata, LintingRuleConfig>; From 9a32ace9080d1913213050e7f7baa5b620035aa4 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 5 Sep 2025 14:55:25 +0200 Subject: [PATCH 46/70] doc(repl): improve comments --- src/cli/repl/commands/repl-main.ts | 17 ++++++++++++++--- src/cli/repl/commands/repl-parse.ts | 1 + src/cli/repl/commands/repl-query.ts | 5 +++++ src/util/collections/arraymap.ts | 5 ++--- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/cli/repl/commands/repl-main.ts b/src/cli/repl/commands/repl-main.ts index 34a8251ae4d..2cb89fad80d 100644 --- a/src/cli/repl/commands/repl-main.ts +++ b/src/cli/repl/commands/repl-main.ts @@ -28,10 +28,10 @@ export const standardReplOutput: ReplOutput = { stderr: console.error }; + /** - * Information passed to each repl command function + * Information passed to each {@link ReplCommand#fn}. */ - export interface ReplCommandInformation { output: ReplOutput, allowRSessionAccess: boolean, @@ -40,6 +40,11 @@ export interface ReplCommandInformation { config: FlowrConfigOptions } + +/** + * Information passed to each {@link ReplCodeCommand#fn}. + * The {@link analyzer} has the {@link RParseRequest}. + */ export interface ReplCodeCommandInformation { output: ReplOutput, analyzer: FlowrAnalyzer @@ -70,6 +75,10 @@ export interface ReplCommand extends ReplBaseCommand { fn: (info: ReplCommandInformation) => Promise | void } + +/** + * Repl command that uses the {@link FlowrAnalyzer} + */ export interface ReplCodeCommand extends ReplBaseCommand { usesAnalyzer: true; /** @@ -77,6 +86,8 @@ export interface ReplCodeCommand extends ReplBaseCommand { * Furthermore, it has to obey the formatter defined in the {@link ReplOutput}. */ fn: (info: ReplCodeCommandInformation) => Promise | void - /** Argument parser */ + /** + * Argument parser function which handles the input given after the repl command + */ argsParser: (remainingLine: string) => { input: string, remaining: string[]} } diff --git a/src/cli/repl/commands/repl-parse.ts b/src/cli/repl/commands/repl-parse.ts index cc9cf6a244a..4e90ecc922f 100644 --- a/src/cli/repl/commands/repl-parse.ts +++ b/src/cli/repl/commands/repl-parse.ts @@ -160,6 +160,7 @@ export const parseCommand: ReplCodeCommand = { script: false, argsParser: (line: string) => { return { + // Threat the whole input line as R code input: removeRQuotes(line.trim()), remaining: [] }; diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index 6e8fb54d36f..3793f85d688 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -76,6 +76,11 @@ async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalyzer, rem }; } +/** + * Function for splitting the input line. + * The first token is the query command. + * The rest of the line is treated as input code. + */ function parseArgs(line: string) { const args = splitAtEscapeSensitive(line); const command = args.shift(); diff --git a/src/util/collections/arraymap.ts b/src/util/collections/arraymap.ts index 68f44826e94..addf5ed7488 100644 --- a/src/util/collections/arraymap.ts +++ b/src/util/collections/arraymap.ts @@ -9,15 +9,14 @@ export class ArrayMap { } /** - * Sets a value for a given key. + * Sets a value for a given key array. */ public set(key: readonly K[], v: V): void { this.internal.set(this.makeKey(key), v); } /** - * Return a value for the given key, if the key does not exist within the default map, - * this will invoke the generator and assign the produced value. + * Return the value for the key array. */ public get(key: readonly K[]): V | undefined { return this.internal.get(this.makeKey(key)); From 5fd85f5b18e8e93b9f46b7ad72dd02109715cbce Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Mon, 8 Sep 2025 16:17:22 +0200 Subject: [PATCH 47/70] feat-fix(query): resolve promise Stupid beginner mistake :( --- src/cli/repl/commands/repl-query.ts | 4 ++-- src/queries/catalog/config-query/config-query-executor.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index 3793f85d688..9485b547355 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -67,10 +67,10 @@ async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalyzer, rem const dummyProject = await getDummyFlowrProject(); return { - query: executeQueries({ + query: await Promise.resolve(executeQueries({ input: analyzer, libraries: dummyProject.libraries }, - parsedQuery), + parsedQuery)), parsedQuery, processed: { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalizedAst() } }; diff --git a/src/queries/catalog/config-query/config-query-executor.ts b/src/queries/catalog/config-query/config-query-executor.ts index a1a71c0097e..255899a6c10 100644 --- a/src/queries/catalog/config-query/config-query-executor.ts +++ b/src/queries/catalog/config-query/config-query-executor.ts @@ -16,13 +16,13 @@ export function executeConfigQuery({ input }: BasicQueryData, queries: readonly deepMergeObject(config, update); } - return new Promise(() => { - return { + return new Promise((resolve) => { + resolve({ '.meta': { /* there is no sense in measuring a get */ timing: 0 }, config: config - }; + }); }); } From 0939abf7ae4de8d566379cd1ddd6975ab907915b Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:09:26 +0200 Subject: [PATCH 48/70] feat(analyzer): use analyzer for queries --- src/cli/repl/server/connection.ts | 3 +-- src/documentation/doc-util/doc-query.ts | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index 6bd921c5aad..21f02cf391d 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -40,7 +40,6 @@ import { requestLineageMessage } from './messages/message-lineage'; import { getLineage } from '../commands/repl-lineage'; import type { QueryRequestMessage, QueryResponseMessage } from './messages/message-query'; import { requestQueryMessage } from './messages/message-query'; -import { executeQueries } from '../../../queries/query'; import type { KnownParser, ParseStepOutput } from '../../../r-bridge/parser'; import { compact } from './compact'; import type { ControlFlowInformation } from '../../../control-flow/control-flow-graph'; @@ -369,7 +368,7 @@ export class FlowRServerConnection { return; } - void Promise.resolve(executeQueries({ input: fileInformation.analyzer }, request.query)).then(results => { + void Promise.resolve(fileInformation.analyzer.query(request.query)).then(results => { sendMessage(this.socket, { type: 'response-query', id: request.id, diff --git a/src/documentation/doc-util/doc-query.ts b/src/documentation/doc-util/doc-query.ts index de683959c5a..0cef65f4088 100644 --- a/src/documentation/doc-util/doc-query.ts +++ b/src/documentation/doc-util/doc-query.ts @@ -1,6 +1,5 @@ import type { RShell } from '../../r-bridge/shell'; import type { Queries, SupportedQueryTypes } from '../../queries/query'; -import { executeQueries } from '../../queries/query'; import { requestFromInput } from '../../r-bridge/retriever'; import { jsonReplacer } from '../../util/json'; import { markdownFormatter } from '../../util/text/ansi'; @@ -26,7 +25,7 @@ export async function showQuery< >(shell: RShell, code: string, queries: Queries, { showCode, collapseResult, collapseQuery }: ShowQueryOptions = {}): Promise { const now = performance.now(); const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)).setParser(shell).build(); - const results = executeQueries({ input: analyzer }, queries); + const results = await analyzer.query(queries); const duration = performance.now() - now; const metaInfo = ` From 298c658fb9a7a692be5602ae1672d27f1684cfa6 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Mon, 8 Sep 2025 18:26:53 +0200 Subject: [PATCH 49/70] feat-fix: use more specific type --- .../catalog/dependencies-query/dependencies-query-executor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts index d5f5e129422..c5f4491193d 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts @@ -73,7 +73,7 @@ export async function executeDependenciesQuery({ } as DependenciesQueryResult; } -function makeCallContextQuery(functions: readonly FunctionInfo[], kind: string): CallContextQuery[] { +function makeCallContextQuery(functions: readonly FunctionInfo[], kind: DependencyCategoryName): CallContextQuery[] { return functions.map(f => ({ type: 'call-context', callName: f.name, From 48006661cc676656bdbab8c6b1bd8bed796ff45a Mon Sep 17 00:00:00 2001 From: stimjannik Date: Wed, 10 Sep 2025 17:33:00 +0200 Subject: [PATCH 50/70] feat-fix(libraries): include libraries in new dependencies query --- .../dependencies-query-executor.ts | 16 +++++++++------- .../dependencies-query-format.ts | 16 ++++++++++------ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts index c5f4491193d..ae30914d008 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts @@ -59,7 +59,7 @@ export async function executeDependenciesQuery({ await executeQueriesOfSameType(data, functions.entries().map(([c, f]) => makeCallContextQuery(f, c)).toArray().flat()); const results = Object.fromEntries(functions.entries().map(([c, f]) => { - const results = getResults(queries, { dataflow, config, normalize }, queryResults, c, f); + const results = getResults(queries, { dataflow, config, normalize }, queryResults, c, f, data); // only default categories allow additional analyses, so we null coalese here! (DefaultDependencyCategories as Record)[c]?.additionalAnalysis?.(data, ignoreDefault, f, queryResults, results); return [c, results]; @@ -96,7 +96,7 @@ function dropInfoOnLinkedIds(linkedIds: readonly (NodeId | { id: NodeId, info: o const readOnlyModes = new Set(['r', 'rt', 'rb']); const writeOnlyModes = new Set(['w', 'wt', 'wb', 'a', 'at', 'ab']); -function getResults(queries: readonly DependenciesQuery[], { dataflow, config, normalize }: { dataflow: DataflowInformation, config: FlowrConfigOptions, normalize: NormalizedAst }, results: CallContextQueryResult, kind: DependencyCategoryName, functions: FunctionInfo[]): DependencyInfo[] { +function getResults(queries: readonly DependenciesQuery[], { dataflow, config, normalize }: { dataflow: DataflowInformation, config: FlowrConfigOptions, normalize: NormalizedAst }, results: CallContextQueryResult, kind: DependencyCategoryName, functions: FunctionInfo[], data?: BasicQueryData): DependencyInfo[] { const defaultValue = getAllCategories(queries)[kind].defaultValue; const functionMap = new Map(functions.map(f => [f.name, f])); const kindEntries = Object.entries(results?.kinds[kind]?.subkinds ?? {}); @@ -139,11 +139,13 @@ function getResults(queries: readonly DependenciesQuery[], { dataflow, config, n for(const [arg, values] of foundValues.entries()) { for(const value of values) { const result = compactRecord({ - nodeId: id, - functionName: vertex.name, - lexemeOfArgument: getLexeme(value, arg), - linkedIds: linked?.length ? linked : undefined, - value: value ?? defaultValue + nodeId: id, + functionName: vertex.name, + lexemeOfArgument: getLexeme(value, arg), + linkedIds: linked?.length ? linked : undefined, + value: value ?? defaultValue, + versionConstraints: data?.libraries?.find(f => f.name === value)?.versionConstraints ?? undefined, + derivedVersion: data?.libraries?.find(f => f.name === value)?.derivedVersion ?? undefined, } as DependencyInfo); if(result) { results.push(result); diff --git a/src/queries/catalog/dependencies-query/dependencies-query-format.ts b/src/queries/catalog/dependencies-query/dependencies-query-format.ts index 5b310733276..e70e6c29563 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-format.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-format.ts @@ -14,6 +14,7 @@ import { VisualizeFunctions } from './function-info/visualize-functions'; import { visitAst } from '../../../r-bridge/lang-4.x/ast/model/processing/visitor'; import { RType } from '../../../r-bridge/lang-4.x/ast/model/type'; import type { CallContextQueryResult } from '../call-context-query/call-context-query-format'; +import type { Range } from 'semver'; export const Unknown = 'unknown'; @@ -39,6 +40,7 @@ export const DefaultDependencyCategories = { nodeId: n.info.id, functionName: (n.info.fullLexeme ?? n.lexeme).includes(':::') ? ':::' : '::', value: n.namespace, + libraryInfo: data.libraries ?? undefined, }); } }); @@ -79,13 +81,15 @@ export type DependenciesQueryResult = BaseQueryResult & { [C in DefaultDependenc export interface DependencyInfo extends Record{ - nodeId: NodeId - functionName: string - linkedIds?: readonly NodeId[] + nodeId: NodeId + functionName: string + linkedIds?: readonly NodeId[] /** the lexeme is presented whenever the specific info is of {@link Unknown} */ - lexemeOfArgument?: string; + lexemeOfArgument?: string; /** The library name, file, source, destination etc. being sourced, read from, or written to. */ - value?: string + value?: string + versionConstraints?: Range[], + derivedVersion?: Range } function printResultSection(title: string, infos: DependencyInfo[], result: string[]): void { @@ -104,7 +108,7 @@ function printResultSection(title: string, infos: DependencyInfo[], result: stri }, new Map()); for(const [functionName, infos] of grouped) { result.push(` â•° \`${functionName}\``); - result.push(infos.map(i => ` â•° Node Id: ${i.nodeId}${i.value !== undefined ? `, \`${i.value}\`` : ''}`).join('\n')); + result.push(infos.map(i => ` â•° Node Id: ${i.nodeId}${i.value !== undefined ? `, \`${i.value}\`` : ''}${i.derivedVersion !== undefined ? `, Version: \`${i.derivedVersion.toString()}\`` : ''}`).join('\n')); } } From e203f750c45775d0309680b02519b979313d9e05 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 12 Sep 2025 09:06:05 +0200 Subject: [PATCH 51/70] feat(analyzer): add documentation comments --- src/project/flowr-analyzer-builder.ts | 40 ++++++++++++++++++++++ src/project/flowr-analyzer.ts | 49 ++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 08c297bffc9..bfbc0228314 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -8,6 +8,9 @@ import type { KnownParser } from '../r-bridge/parser'; import type { FlowrAnalyzerPlugin } from './plugins/flowr-analyzer-plugin'; import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize'; +/** + * Builder for the {@link FlowrAnalyzer}. + */ export class FlowrAnalyzerBuilder { private flowrConfig: DeepWritable = cloneConfig(defaultConfigOptions); private parser?: KnownParser; @@ -15,47 +18,84 @@ export class FlowrAnalyzerBuilder { private input?: Omit; private plugins: FlowrAnalyzerPlugin[]; + /** + * Apply an amendment to the configuration the builder currently holds. + * Per default, the {@link defaultConfigOptions} are used. + * @param func - Receives the current configuration of the builder and allows for amendment. + */ // eslint-disable-next-line @typescript-eslint/no-invalid-void-type public amendConfig(func: (config: DeepWritable) => FlowrConfigOptions | void): this { this.flowrConfig = amendConfig(this.flowrConfig, func); return this; } + /** + * Overwrite the configuration used by the resulting analyzer. + * @param config - The new configuration. + */ public setConfig(config: FlowrConfigOptions) { this.flowrConfig = config; return this; } + /** + * Set the parser instance used by the analyzer. + * @param parser - The parser. + */ public setParser(parser: KnownParser) { this.parser = parser; return this; } + /** + * Set the engine and hence the parser that will be used by the analyzer. + * @param engine - The engine to use. + */ public setEngine(engine: EngineConfig['type']) { this.flowrConfig.defaultEngine = engine; return this; } + /** + * Additional parameters for the analyses. + * @param input - The input. + */ public setInput(input: Omit) { this.input = input; return this; } + /** + * Create a new builder instance. + * @param request - The code to analyze. + * @param plugins - The plugins to register. + */ constructor(request: RParseRequests, plugins?: FlowrAnalyzerPlugin[]) { this.request = request; this.plugins = plugins ?? []; } + /** + * Register one or multiple additional plugins. + * @param plugin - One or multiple plugins. + */ public registerPlugin(...plugin: FlowrAnalyzerPlugin[]): this { this.plugins.push(...plugin); return this; } + /** + * Remove one or multiple plugins. + * @param plugin - One or multiple plugins. + */ public unregisterPlugin(...plugin: FlowrAnalyzerPlugin[]): this { this.plugins = this.plugins.filter(p => !plugin.includes(p)); return this; } + /** + * Create the {@link FlowrAnalyzer} instance using the given information. + */ public async build(): Promise { let parser: KnownParser; if(this.parser) { diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 0ebd7d01a8e..be9e53ad813 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -17,6 +17,9 @@ import type { PipelinePerStepMetaInformation } from '../core/steps/pipeline/pipe import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize'; import { ArrayMap } from '../util/collections/arraymap'; +/** + * Exposes the central analyses and information provided by the {@link FlowrAnalyzer} to the linter, search, and query APIs + */ export type FlowrAnalysisInput = { normalizedAst(force?: boolean): Promise; dataflow(force?: boolean): Promise; @@ -29,6 +32,10 @@ interface ControlFlowCache { quick: ControlFlowInformation } +/** + * Central class for creating analyses in FlowR. + * Use the {@link FlowrAnalyzerBuilder} to create a new instance. + */ export class FlowrAnalyzer { public readonly flowrConfig: FlowrConfigOptions; private readonly request: RParseRequests; @@ -43,6 +50,14 @@ export class FlowrAnalyzer { quick: undefined as unknown as ControlFlowInformation }; + /** + * Create a new analyzer instance. + * Prefer the use of the {@link FlowrAnalyzerBuilder} instead of calling this constructor directly. + * @param config - The FlowR config to use for the analyses + * @param parser - The parser to use for parsing the given request. + * @param request - The code to analyze. + * @param requiredInput - Additional parameters used for the analyses. + */ constructor(config: FlowrConfigOptions, parser: KnownParser, request: RParseRequests, requiredInput: Omit) { this.flowrConfig = config; this.request = request; @@ -63,6 +78,11 @@ export class FlowrAnalyzer { return this.parser.name; } + /** + * Get the parse output for the request. + * The parse result type depends on the {@link KnownParser} used by the analyzer. + * @param force - Do not use the cache, instead force a new parse. + */ // TODO TSchoeller Fix type public async parseOutput(force?: boolean): Promise & PipelinePerStepMetaInformation> { if(this.parse && !force) { @@ -82,6 +102,10 @@ export class FlowrAnalyzer { return result.parse; } + /** + * Get the normalized abstract syntax tree for the request. + * @param force - Do not use the cache, instead force new analyses. + */ public async normalizedAst(force?: boolean): Promise { if(this.ast && !force) { return { @@ -100,6 +124,10 @@ export class FlowrAnalyzer { return result.normalize; } + /** + * Get the dataflow graph for the request. + * @param force - Do not use the cache, instead force new analyses. + */ public async dataflow(force?: boolean): Promise { if(this.dataflowInfo && !force) { return { @@ -119,6 +147,12 @@ export class FlowrAnalyzer { return result.dataflow; } + /** + * Get the control flow graph (CFG) for the request. + * @param simplifications - Simplification passes to be applied to the CFG. + * @param useDataflow - Whether to use the dataflow graph for the creation of the CFG. + * @param force - Do not use the cache, instead force new analyses. + */ public async controlFlow(simplifications?: readonly CfgSimplificationPassName[], useDataflow?: boolean, force?: boolean): Promise { if(!force) { const value = this.controlFlowInfos.simplified.get(simplifications ?? []); @@ -140,6 +174,10 @@ export class FlowrAnalyzer { return result; } + /** + * Get a more performant version of the control flow graph. + * @param force - Do not use the cache, instead force new analyses. + */ public async controlFlowQuick(force?: boolean): Promise { if(!force) { if(this.controlFlowInfos.quick) { @@ -162,10 +200,11 @@ export class FlowrAnalyzer { return result; } - public async query(query: Queries, force?: boolean) { - if(!this.dataflowInfo) { - await this.dataflow(force); - } - return executeQueries({ input: this }, query); + /** + * Access the query API for the request. + * @param query - The list of queries. + */ + public async query(query: Queries) { + return await Promise.resolve(executeQueries({ input: this }, query)); } } \ No newline at end of file From 72c1269a5793e6007bcba5ef8dcc260414804d13 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 12 Sep 2025 09:21:46 +0200 Subject: [PATCH 52/70] feat-fix(analyzer): rename ArrayMap Reflects actual generality of the approach --- src/project/flowr-analyzer.ts | 8 ++++---- src/util/collections/{arraymap.ts => objectmap.ts} | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) rename src/util/collections/{arraymap.ts => objectmap.ts} (53%) diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index be9e53ad813..33e9e911aae 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -15,7 +15,7 @@ import type { DataflowInformation } from '../dataflow/info'; import type { CfgSimplificationPassName } from '../control-flow/cfg-simplification'; import type { PipelinePerStepMetaInformation } from '../core/steps/pipeline/pipeline'; import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize'; -import { ArrayMap } from '../util/collections/arraymap'; +import { ObjectMap } from '../util/collections/objectmap'; /** * Exposes the central analyses and information provided by the {@link FlowrAnalyzer} to the linter, search, and query APIs @@ -28,7 +28,7 @@ export type FlowrAnalysisInput = { } interface ControlFlowCache { - simplified: ArrayMap, + simplified: ObjectMap, quick: ControlFlowInformation } @@ -46,7 +46,7 @@ export class FlowrAnalyzer { private ast = undefined as unknown as NormalizedAst; private dataflowInfo = undefined as unknown as DataflowInformation; private controlFlowInfos: ControlFlowCache = { - simplified: new ArrayMap(), + simplified: new ObjectMap(), quick: undefined as unknown as ControlFlowInformation }; @@ -69,7 +69,7 @@ export class FlowrAnalyzer { this.ast = undefined as unknown as NormalizedAst; this.dataflowInfo = undefined as unknown as DataflowInformation; this.controlFlowInfos = { - simplified: new ArrayMap(), + simplified: new ObjectMap(), quick: undefined as unknown as ControlFlowInformation }; } diff --git a/src/util/collections/arraymap.ts b/src/util/collections/objectmap.ts similarity index 53% rename from src/util/collections/arraymap.ts rename to src/util/collections/objectmap.ts index addf5ed7488..ed02626636c 100644 --- a/src/util/collections/arraymap.ts +++ b/src/util/collections/objectmap.ts @@ -1,7 +1,9 @@ /** - * A map type that uses an array of strings as key. + * A map type that accepts an arbitrary object as key. + * {@link JSON.stringify} is used to create the actual key for the underlying map. + * This can be helpful if value equality is desired. */ -export class ArrayMap { +export class ObjectMap { private readonly internal = new Map(); private makeKey(key: readonly K[]): string { @@ -9,14 +11,14 @@ export class ArrayMap { } /** - * Sets a value for a given key array. + * Sets a value for a given key. */ public set(key: readonly K[], v: V): void { this.internal.set(this.makeKey(key), v); } /** - * Return the value for the key array. + * Return the value for the key. */ public get(key: readonly K[]): V | undefined { return this.internal.get(this.makeKey(key)); From b20adc4355a6fc36dc9a9bec7e79afd3361be949 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 12 Sep 2025 10:49:56 +0200 Subject: [PATCH 53/70] feat-fix(repl): fix config updates --- src/cli/repl/commands/repl-query.ts | 5 ++--- src/queries/catalog/config-query/config-query-executor.ts | 8 +++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index 9485b547355..f6926ff6057 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -25,7 +25,7 @@ function printHelp(output: ReplOutput) { } async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalyzer, remainingArgs: string[]): Promise, processed: {dataflow: DataflowInformation, normalize: NormalizedAst} }> { - const query = remainingArgs[0]; + const query = remainingArgs.shift(); if(!query) { output.stderr('No query provided, use \':query help\' to get more information.'); @@ -83,10 +83,9 @@ async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalyzer, rem */ function parseArgs(line: string) { const args = splitAtEscapeSensitive(line); - const command = args.shift(); return { input: args.join(' ').trim(), - remaining: command ? [command] : [] + remaining: args }; } diff --git a/src/queries/catalog/config-query/config-query-executor.ts b/src/queries/catalog/config-query/config-query-executor.ts index 255899a6c10..f3692a0c1e3 100644 --- a/src/queries/catalog/config-query/config-query-executor.ts +++ b/src/queries/catalog/config-query/config-query-executor.ts @@ -2,18 +2,16 @@ import { log } from '../../../util/log'; import type { ConfigQuery, ConfigQueryResult } from './config-query-format'; import type { BasicQueryData } from '../../base-query-format'; import { isNotUndefined } from '../../../util/assert'; -import { deepMergeObject } from '../../../util/objects'; -import { cloneConfig } from '../../../config'; +import { deepMergeObjectInPlace } from '../../../util/objects'; export function executeConfigQuery({ input }: BasicQueryData, queries: readonly ConfigQuery[]): Promise { if(queries.length !== 1) { log.warn('Config query usually expects only up to one query, but got', queries.length); } const updates = queries.map(q => q.update).filter(isNotUndefined); - const config = cloneConfig(input.flowrConfig); for(const update of updates) { - deepMergeObject(config, update); + deepMergeObjectInPlace(input.flowrConfig, update); } return new Promise((resolve) => { @@ -22,7 +20,7 @@ export function executeConfigQuery({ input }: BasicQueryData, queries: readonly /* there is no sense in measuring a get */ timing: 0 }, - config: config + config: input.flowrConfig }); }); } From 04e92cb48c3fa71b58391073047555facb563dee Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 12 Sep 2025 11:01:20 +0200 Subject: [PATCH 54/70] feat-fix(analyzer): proposal to generalize analyzer over parser --- src/cli/repl/commands/repl-cfg.ts | 4 ++-- src/cli/repl/commands/repl-main.ts | 4 ++-- src/cli/repl/commands/repl-query.ts | 4 ++-- src/project/flowr-analyzer-builder.ts | 2 +- src/project/flowr-analyzer.ts | 11 +++++------ .../flowr-analyzer-description-file-plugin.ts | 6 +++--- src/project/plugins/flowr-analyzer-plugin.ts | 6 +++--- ...-analyzer-loading-order-description-file-plugin.ts | 4 ++-- ...alyzer-package-versions-description-file-plugin.ts | 4 ++-- 9 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/cli/repl/commands/repl-cfg.ts b/src/cli/repl/commands/repl-cfg.ts index f1230f8b656..94a1b915748 100644 --- a/src/cli/repl/commands/repl-cfg.ts +++ b/src/cli/repl/commands/repl-cfg.ts @@ -6,14 +6,14 @@ import type { ControlFlowInformation } from '../../../control-flow/control-flow- import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { CfgSimplificationPassName } from '../../../control-flow/cfg-simplification'; import { DefaultCfgSimplificationOrder } from '../../../control-flow/cfg-simplification'; -import type { FlowrAnalyzer } from '../../../project/flowr-analyzer'; +import type { FlowrAnalysisInput } from '../../../project/flowr-analyzer'; import { handleString } from '../core'; function formatInfo(out: ReplOutput, type: string): string { return out.formatter.format(`Copied ${type} to clipboard.`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); } -async function produceAndPrintCfg(analyzer: FlowrAnalyzer, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string) { +async function produceAndPrintCfg(analyzer: FlowrAnalysisInput, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string) { const cfg = await analyzer.controlFlow([...DefaultCfgSimplificationOrder, ...simplifications]); const result = await analyzer.normalizedAst(); const mermaid = cfgConverter(cfg, result); diff --git a/src/cli/repl/commands/repl-main.ts b/src/cli/repl/commands/repl-main.ts index 2cb89fad80d..29e2c755ed2 100644 --- a/src/cli/repl/commands/repl-main.ts +++ b/src/cli/repl/commands/repl-main.ts @@ -2,7 +2,7 @@ import type { OutputFormatter } from '../../../util/text/ansi'; import { formatter } from '../../../util/text/ansi'; import type { KnownParser } from '../../../r-bridge/parser'; import type { FlowrConfigOptions } from '../../../config'; -import type { FlowrAnalyzer } from '../../../project/flowr-analyzer'; +import type { FlowrAnalysisInput } from '../../../project/flowr-analyzer'; /** * Defines the main interface for output of the repl. @@ -47,7 +47,7 @@ export interface ReplCommandInformation { */ export interface ReplCodeCommandInformation { output: ReplOutput, - analyzer: FlowrAnalyzer + analyzer: FlowrAnalysisInput remainingArgs: string[] } diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index f6926ff6057..ff9c87b28b9 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -7,7 +7,7 @@ import type { Query, QueryResults, SupportedQuery, SupportedQueryTypes } from '. import { AnyQuerySchema, executeQueries, QueriesSchema, SupportedQueries } from '../../../queries/query'; import { jsonReplacer } from '../../../util/json'; import { asciiSummaryOfQueryResult } from '../../../queries/query-print'; -import type { FlowrAnalyzer } from '../../../project/flowr-analyzer'; +import type { FlowrAnalysisInput } from '../../../project/flowr-analyzer'; import { getDummyFlowrProject } from '../../../project/flowr-project'; import type { DataflowInformation } from '../../../dataflow/info'; import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; @@ -24,7 +24,7 @@ function printHelp(output: ReplOutput) { output.stdout(`With this, ${italic(':query @config', output.formatter)} prints the result of the config query.`); } -async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalyzer, remainingArgs: string[]): Promise, processed: {dataflow: DataflowInformation, normalize: NormalizedAst} }> { +async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisInput, remainingArgs: string[]): Promise, processed: {dataflow: DataflowInformation, normalize: NormalizedAst} }> { const query = remainingArgs.shift(); if(!query) { diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index bfbc0228314..2903d94808c 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -96,7 +96,7 @@ export class FlowrAnalyzerBuilder { /** * Create the {@link FlowrAnalyzer} instance using the given information. */ - public async build(): Promise { + public async build(): Promise> { let parser: KnownParser; if(this.parser) { parser = this.parser; diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 33e9e911aae..71e02cfd066 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -36,13 +36,13 @@ interface ControlFlowCache { * Central class for creating analyses in FlowR. * Use the {@link FlowrAnalyzerBuilder} to create a new instance. */ -export class FlowrAnalyzer { +export class FlowrAnalyzer { public readonly flowrConfig: FlowrConfigOptions; private readonly request: RParseRequests; - private readonly parser: KnownParser; + private readonly parser: Parser; private readonly requiredInput: Omit; - private parse = undefined as unknown as ParseStepOutput; + private parse = undefined as unknown as ParseStepOutput>>; private ast = undefined as unknown as NormalizedAst; private dataflowInfo = undefined as unknown as DataflowInformation; private controlFlowInfos: ControlFlowCache = { @@ -58,7 +58,7 @@ export class FlowrAnalyzer { * @param request - The code to analyze. * @param requiredInput - Additional parameters used for the analyses. */ - constructor(config: FlowrConfigOptions, parser: KnownParser, request: RParseRequests, requiredInput: Omit) { + constructor(config: FlowrConfigOptions, parser: Parser, request: RParseRequests, requiredInput: Omit) { this.flowrConfig = config; this.request = request; this.parser = parser; @@ -83,8 +83,7 @@ export class FlowrAnalyzer { * The parse result type depends on the {@link KnownParser} used by the analyzer. * @param force - Do not use the cache, instead force a new parse. */ - // TODO TSchoeller Fix type - public async parseOutput(force?: boolean): Promise & PipelinePerStepMetaInformation> { + public async parseOutput(force?: boolean): Promise>> & PipelinePerStepMetaInformation> { if(this.parse && !force) { return { ...this.parse, diff --git a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts index 82e0d4218ce..82bd955eb09 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts @@ -1,8 +1,8 @@ import { FlowrAnalyzerFilePlugin } from './flowr-analyzer-file-plugin'; import { SemVer } from 'semver'; -import type { FlowrAnalyzer } from '../../flowr-analyzer'; +import type { FlowrAnalysisInput } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; -import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; +import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import { parseDCF } from '../../../util/files'; export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin { @@ -12,7 +12,7 @@ export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin public readonly dependencies: FlowrAnalyzerPlugin[] = []; public information: Map = new Map(); - public async processor(_analyzer: FlowrAnalyzer, _pluginConfig: FlowrConfigOptions): Promise { + public async processor(_analyzer: FlowrAnalysisInput, _pluginConfig: FlowrConfigOptions): Promise { if(this.files.length === 0) { throw new Error('FlowrAnalyzerDescriptionFilePlugin: No DESCRIPTION file found.'); } diff --git a/src/project/plugins/flowr-analyzer-plugin.ts b/src/project/plugins/flowr-analyzer-plugin.ts index d9c8438b774..34243668c54 100644 --- a/src/project/plugins/flowr-analyzer-plugin.ts +++ b/src/project/plugins/flowr-analyzer-plugin.ts @@ -1,5 +1,5 @@ import type { SemVer } from 'semver'; -import type { FlowrAnalyzer } from '../flowr-analyzer'; +import type { FlowrAnalysisInput } from '../flowr-analyzer'; import type { FlowrConfigOptions } from '../../config'; import type { PathLike } from 'fs'; @@ -12,7 +12,7 @@ export interface FlowrAnalyzerPluginInterface { readonly type: PluginType; dependencies: FlowrAnalyzerPlugin[]; - processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise; + processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): Promise; } export abstract class FlowrAnalyzerPlugin implements FlowrAnalyzerPluginInterface { @@ -27,6 +27,6 @@ export abstract class FlowrAnalyzerPlugin implements FlowrAnalyzerPluginInterfac this.rootPath = rootPath; } - public abstract processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise; + public abstract processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): Promise; } diff --git a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts index be93e6105e1..d9749a4f4be 100644 --- a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts +++ b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts @@ -1,7 +1,7 @@ import { FlowrAnalyzerDescriptionFilePlugin } from '../file-plugins/flowr-analyzer-description-file-plugin'; import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import { SemVer } from 'semver'; -import type { FlowrAnalyzer } from '../../flowr-analyzer'; +import type { FlowrAnalysisInput } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; import { FlowrAnalyzerLoadingOrderPlugin } from './flowr-analyzer-loading-order-plugin'; @@ -13,7 +13,7 @@ export class FlowrAnalyzerLoadingOrderDescriptionFilePlugin extends FlowrAnalyze dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; descriptionFile: Map = new Map(); - async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { + async processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): Promise { const plugin = this.dependencies[0] as FlowrAnalyzerDescriptionFilePlugin; await plugin.processor(analyzer, pluginConfig); this.descriptionFile = plugin.information; diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts index 7384677009e..79ec77609e3 100644 --- a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts @@ -2,7 +2,7 @@ import { FlowrAnalyzerPackageVersionsPlugin } from './flowr-analyzer-package-ver import { FlowrAnalyzerDescriptionFilePlugin } from '../file-plugins/flowr-analyzer-description-file-plugin'; import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import { SemVer } from 'semver'; -import type { FlowrAnalyzer } from '../../flowr-analyzer'; +import type { FlowrAnalysisInput } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; import type { PackageType } from './package'; import { Package } from './package'; @@ -15,7 +15,7 @@ export class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnal dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; descriptionFile: Map = new Map(); - async processor(analyzer: FlowrAnalyzer, pluginConfig: FlowrConfigOptions): Promise { + async processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): Promise { const plugin = this.dependencies[0] as FlowrAnalyzerDescriptionFilePlugin; await plugin.processor(analyzer, pluginConfig); this.descriptionFile = plugin.information; From 118be5201641fc91a2a4041bf8592bfca13deb14 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Tue, 23 Sep 2025 22:09:38 +0200 Subject: [PATCH 55/70] feat-fix(libraries): versions for ':::' and '::' reduced to necessary --- .../dependencies-query/dependencies-query-format.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/queries/catalog/dependencies-query/dependencies-query-format.ts b/src/queries/catalog/dependencies-query/dependencies-query-format.ts index e70e6c29563..cde4a3bdb48 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-format.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-format.ts @@ -37,10 +37,11 @@ export const DefaultDependencyCategories = { if(n.type === RType.Symbol && n.namespace) { /* we should improve the identification of ':::' */ result.push({ - nodeId: n.info.id, - functionName: (n.info.fullLexeme ?? n.lexeme).includes(':::') ? ':::' : '::', - value: n.namespace, - libraryInfo: data.libraries ?? undefined, + nodeId: n.info.id, + functionName: (n.info.fullLexeme ?? n.lexeme).includes(':::') ? ':::' : '::', + value: n.namespace, + versionConstraints: data?.libraries?.find(f => f.name === n.namespace)?.versionConstraints ?? undefined, + derivedVersion: data?.libraries?.find(f => f.name === n.namespace)?.derivedVersion ?? undefined, }); } }); From af6ec545d8741a27204d1b6fee211ca39f229e11 Mon Sep 17 00:00:00 2001 From: stimjannik Date: Tue, 23 Sep 2025 22:10:24 +0200 Subject: [PATCH 56/70] test-fix(libraries): replace old values in expected results --- .../dataflow/query/dependencies-query.test.ts | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/test/functionality/dataflow/query/dependencies-query.test.ts b/test/functionality/dataflow/query/dependencies-query.test.ts index 28557fa2b6c..bb7be4bbb8e 100644 --- a/test/functionality/dataflow/query/dependencies-query.test.ts +++ b/test/functionality/dataflow/query/dependencies-query.test.ts @@ -82,8 +82,8 @@ describe('Dependencies Query', withTreeSitter(parser => { ] }); - testQuery('Library with variable', 'a <- "ggplot2"\nb <- TRUE\nlibrary(a,character.only=b)', { libraries: [ - { nodeId: '3@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] } + testQuery('Library with variable', 'a <- "ggplot2"\nb <- TRUE\nlibrary(a,character.only=b)', { library: [ + { nodeId: '3@library', functionName: 'library', value: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] } ] }); // for now, we want a better or (https://github.com/flowr-analysis/flowr/issues/1342) @@ -163,10 +163,10 @@ describe('Dependencies Query', withTreeSitter(parser => { { nodeId: '3@library', functionName: 'library', value: 'unknown', lexemeOfArgument: 'v' }, ] }); - testQuery('Using a vector by variable (real world)', 'packages <- c("ggplot2", "dplyr", "tidyr")\nlapply(packages, library, character.only = TRUE)', { libraries: [ - { nodeId: '2@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] }, - { nodeId: '2@library', functionName: 'library', libraryName: 'dplyr', derivedVersion: new Range('>=1.4.0'), versionConstraints: [new Range('>=1.4.0')] }, - { nodeId: '2@library', functionName: 'library', libraryName: 'tidyr' } + testQuery('Using a vector by variable (real world)', 'packages <- c("ggplot2", "dplyr", "tidyr")\nlapply(packages, library, character.only = TRUE)', { library: [ + { nodeId: '2@library', functionName: 'library', value: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] }, + { nodeId: '2@library', functionName: 'library', value: 'dplyr', derivedVersion: new Range('>=1.4.0'), versionConstraints: [new Range('>=1.4.0')] }, + { nodeId: '2@library', functionName: 'library', value: 'tidyr' } ] }); testQuery('Using a deeply nested vector by variable', 'v <- c(c(c("a", c("b")), "c"), "d", c("e", c("f", "g")))\nlapply(v, library, character.only = TRUE)', { library: [ @@ -179,19 +179,19 @@ describe('Dependencies Query', withTreeSitter(parser => { { nodeId: '2@library', functionName: 'library', value: 'g' } ] }); - testQuery('Library with version', 'library(ggplot2)', { libraries: [ - { nodeId: '1@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] } + testQuery('Library with version', 'library(ggplot2)', { library: [ + { nodeId: '1@library', functionName: 'library', value: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] } ] }); - testQuery('Libraries with versions', 'library(ggplot2)\nlibrary(dplyr)', { libraries: [ - { nodeId: '1@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] }, - { nodeId: '2@library', functionName: 'library', libraryName: 'dplyr', derivedVersion: new Range('>=1.4.0'), versionConstraints: [new Range('>=1.4.0')] }, + testQuery('Libraries with versions', 'library(ggplot2)\nlibrary(dplyr)', { library: [ + { nodeId: '1@library', functionName: 'library', value: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] }, + { nodeId: '2@library', functionName: 'library', value: 'dplyr', derivedVersion: new Range('>=1.4.0'), versionConstraints: [new Range('>=1.4.0')] }, ] }); - testQuery('Libraries with and without versions', 'library(ggplot2)\nlibrary(dplyr)\nlibrary(tidyr)', { libraries: [ - { nodeId: '1@library', functionName: 'library', libraryName: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] }, - { nodeId: '2@library', functionName: 'library', libraryName: 'dplyr', derivedVersion: new Range('>=1.4.0'), versionConstraints: [new Range('>=1.4.0')] }, - { nodeId: '3@library', functionName: 'library', libraryName: 'tidyr', derivedVersion: undefined, versionConstraints: undefined }, + testQuery('Libraries with and without versions', 'library(ggplot2)\nlibrary(dplyr)\nlibrary(tidyr)', { library: [ + { nodeId: '1@library', functionName: 'library', value: 'ggplot2', derivedVersion: new Range('>=2.5.8'), versionConstraints: [new Range('>=2.5.8')] }, + { nodeId: '2@library', functionName: 'library', value: 'dplyr', derivedVersion: new Range('>=1.4.0'), versionConstraints: [new Range('>=1.4.0')] }, + { nodeId: '3@library', functionName: 'library', value: 'tidyr', derivedVersion: undefined, versionConstraints: undefined }, ] }); describe('Custom', () => { From a1f519ff098afe040507436ebd598ece56e25fb2 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Wed, 24 Sep 2025 09:24:32 +0200 Subject: [PATCH 57/70] feat-fix(server): use then syntax --- src/cli/repl/server/connection.ts | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index 21f02cf391d..79c61f8ac35 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -107,13 +107,13 @@ export class FlowRServerConnection { void this.handleFileAnalysisRequest(request.message as FileAnalysisRequestMessage); break; case 'request-slice': - void this.handleSliceRequest(request.message as SliceRequestMessage); + this.handleSliceRequest(request.message as SliceRequestMessage); break; case 'request-repl-execution': this.handleRepl(request.message as ExecuteRequestMessage); break; case 'request-lineage': - void this.handleLineageRequest(request.message as LineageRequestMessage); + this.handleLineageRequest(request.message as LineageRequestMessage); break; case 'request-query': this.handleQueryRequest(request.message as QueryRequestMessage); @@ -233,7 +233,7 @@ export class FlowRServerConnection { return analyzer; } - private async handleSliceRequest(base: SliceRequestMessage) { + private handleSliceRequest(base: SliceRequestMessage) { const requestResult = validateMessage(base, requestSliceMessage); if(requestResult.type === 'error') { answerForValidationError(this.socket, requestResult, base.id); @@ -254,14 +254,12 @@ export class FlowRServerConnection { return; } - try { - const result = await fileInformation.analyzer.query([{ - type: 'static-slice', - criteria: request.criterion, - noMagicComments: request.noMagicComments, - direction: request.direction - }]); - + void fileInformation.analyzer.query([{ + type: 'static-slice', + criteria: request.criterion, + noMagicComments: request.noMagicComments, + direction: request.direction + }]).then(result => { sendMessage(this.socket, { type: 'response-slice', id: request.id, @@ -270,7 +268,7 @@ export class FlowRServerConnection { .filter(([k,]) => DEFAULT_SLICING_PIPELINE.steps.get(k)?.executed === PipelineStepStage.OncePerRequest) ) as SliceResponseMessage['results'] }); - } catch(e) { + }).catch(e => { this.logger.error(`[${this.name}] Error while analyzing file for token ${request.filetoken}: ${String(e)}`); sendMessage(this.socket, { id: request.id, @@ -278,7 +276,7 @@ export class FlowRServerConnection { fatal: false, reason: `Error while analyzing file for token ${request.filetoken}: ${String(e)}` }); - } + }); } From b890d702a2decbc46a591f0a48450e2722160269 Mon Sep 17 00:00:00 2001 From: MaxAtoms <7847075+MaxAtoms@users.noreply.github.com> Date: Wed, 24 Sep 2025 09:27:12 +0200 Subject: [PATCH 58/70] feat-fix(dataframe): map nodes Previously, the code expected an implicit link between the search result elements and the normalized AST. --- .../rules/dataframe-access-validation.ts | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/linter/rules/dataframe-access-validation.ts b/src/linter/rules/dataframe-access-validation.ts index 9d0bd6bf65e..429b2cddd67 100644 --- a/src/linter/rules/dataframe-access-validation.ts +++ b/src/linter/rules/dataframe-access-validation.ts @@ -1,9 +1,20 @@ -import { type AbstractInterpretationInfo, type DataFrameOperationType, hasDataFrameExpressionInfo } from '../../abstract-interpretation/data-frame/absint-info'; -import { type DataFrameDomain, satisfiesColsNames, satisfiesLeqInterval } from '../../abstract-interpretation/data-frame/domain'; -import { inferDataFrameShapes, resolveIdToDataFrameShape } from '../../abstract-interpretation/data-frame/shape-inference'; +import { + type AbstractInterpretationInfo, + type DataFrameOperationType, + hasDataFrameExpressionInfo +} from '../../abstract-interpretation/data-frame/absint-info'; +import { + type DataFrameDomain, + satisfiesColsNames, + satisfiesLeqInterval +} from '../../abstract-interpretation/data-frame/domain'; +import { + inferDataFrameShapes, + resolveIdToDataFrameShape +} from '../../abstract-interpretation/data-frame/shape-inference'; import { amendConfig } from '../../config'; import { extractCfg } from '../../control-flow/extract-cfg'; -import type { ParentInformation } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { NormalizedAst, ParentInformation } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { NodeId } from '../../r-bridge/lang-4.x/ast/model/processing/node-id'; import { RType } from '../../r-bridge/lang-4.x/ast/model/type'; import type { FlowrSearchElements } from '../../search/flowr-search'; @@ -13,7 +24,7 @@ import { formatRange } from '../../util/mermaid/dfg'; import { type MergeableRecord } from '../../util/objects'; import { rangeFrom, type SourceRange } from '../../util/range'; import type { LintingResult, LintingRule } from '../linter-format'; -import { LintingResultCertainty, LintingPrettyPrintContext, LintingRuleCertainty } from '../linter-format'; +import { LintingPrettyPrintContext, LintingResultCertainty, LintingRuleCertainty } from '../linter-format'; import { LintingRuleTag } from '../linter-tags'; interface DataFrameAccessOperation { @@ -68,7 +79,7 @@ export const DATA_FRAME_ACCESS_VALIDATION = { const cfg = extractCfg(data.normalize, flowrConfig, data.dataflow.graph); inferDataFrameShapes(cfg, data.dataflow.graph, data.normalize, flowrConfig); - const accessOperations = getAccessOperations(elements); + const accessOperations = getAccessOperations(elements, data.normalize); const accesses: DataFrameAccessOperation[] = []; for(const [nodeId, operations] of accessOperations) { @@ -137,10 +148,17 @@ export const DATA_FRAME_ACCESS_VALIDATION = { } as const satisfies LintingRule; function getAccessOperations( - elements: FlowrSearchElements + elements: FlowrSearchElements, + normalize: NormalizedAst ): Map[]> { - return new Map(elements.getElements() - .map(element => element.node) + const nodes = elements.getElements().map( + element => { + const id = element.node.info.id; + return normalize.idMap.get(id); + } + ).filter(n => n !== undefined); + + return new Map(nodes .filter(hasDataFrameExpressionInfo) .map<[NodeId, DataFrameOperationType<'accessCols' | 'accessRows'>[]]>(node => [node.info.id, node.info.dataFrame.operations From 763f679e1b94c725638e874b23ad9848aa592666 Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Thu, 25 Sep 2025 11:12:18 +0200 Subject: [PATCH 59/70] refactor: recover buildability --- src/cli/repl/server/connection.ts | 2 +- src/project/flowr-analyzer.ts | 18 +++++++----- src/project/flowr-project.ts | 2 +- .../config-query/config-query-executor.ts | 14 ++++------ src/queries/query.ts | 28 ++++--------------- .../search-executor/search-enrichers.ts | 6 ++-- .../search-executor/search-generators.ts | 8 ++++-- test/functionality/_helper/query.ts | 21 +++++++++----- 8 files changed, 47 insertions(+), 52 deletions(-) diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index 79c61f8ac35..acf2cb6ccc9 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -178,7 +178,7 @@ export class FlowRServerConnection { id: message.id, cfg: cfg ? cfg2quads(cfg, config()) : undefined, results: { - parse: await printStepResult(PARSE_WITH_R_SHELL_STEP, await analyzer.parseOutput(), StepOutputFormat.RdfQuads, config()), + parse: await printStepResult(PARSE_WITH_R_SHELL_STEP, await analyzer.parseOutput() as ParseStepOutput, StepOutputFormat.RdfQuads, config()), normalize: await printStepResult(NORMALIZE, await analyzer.normalizedAst(), StepOutputFormat.RdfQuads, config()), dataflow: await printStepResult(STATIC_DATAFLOW, await analyzer.dataflow(), StepOutputFormat.RdfQuads, config()) } diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 71e02cfd066..9a032f8f555 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -6,7 +6,7 @@ import { createParsePipeline } from '../core/steps/pipeline/default-pipelines'; import type { KnownParser, ParseStepOutput } from '../r-bridge/parser'; -import type { Queries, SupportedQueryTypes } from '../queries/query'; +import type { Queries, QueryResults, SupportedQueryTypes } from '../queries/query'; import { executeQueries } from '../queries/query'; import { extractCfg, extractCfgQuick } from '../control-flow/extract-cfg'; import type { ControlFlowInformation } from '../control-flow/control-flow-graph'; @@ -21,6 +21,8 @@ import { ObjectMap } from '../util/collections/objectmap'; * Exposes the central analyses and information provided by the {@link FlowrAnalyzer} to the linter, search, and query APIs */ export type FlowrAnalysisInput = { + parserName(): string + parseOutput(force?: boolean): Promise>> & PipelinePerStepMetaInformation> normalizedAst(force?: boolean): Promise; dataflow(force?: boolean): Promise; controlFlow(simplifications?: readonly CfgSimplificationPassName[], useDataflow?: boolean, force?: boolean): Promise; @@ -36,13 +38,13 @@ interface ControlFlowCache { * Central class for creating analyses in FlowR. * Use the {@link FlowrAnalyzerBuilder} to create a new instance. */ -export class FlowrAnalyzer { +export class FlowrAnalyzer { public readonly flowrConfig: FlowrConfigOptions; private readonly request: RParseRequests; private readonly parser: Parser; private readonly requiredInput: Omit; - private parse = undefined as unknown as ParseStepOutput>>; + private parse = undefined as unknown as ParseStepOutput>> & PipelinePerStepMetaInformation; private ast = undefined as unknown as NormalizedAst; private dataflowInfo = undefined as unknown as DataflowInformation; private controlFlowInfos: ControlFlowCache = { @@ -97,8 +99,8 @@ export class FlowrAnalyzer { this.parser, { request: this.request }, this.flowrConfig).allRemainingSteps(); - this.parse = result.parse; - return result.parse; + this.parse = result.parse as unknown as ParseStepOutput>> & PipelinePerStepMetaInformation; + return this.parse; } /** @@ -203,7 +205,9 @@ export class FlowrAnalyzer { * Access the query API for the request. * @param query - The list of queries. */ - public async query(query: Queries) { - return await Promise.resolve(executeQueries({ input: this }, query)); + public async query< + Types extends SupportedQueryTypes = SupportedQueryTypes + >(query: Queries): Promise> { + return executeQueries({ input: this }, query); } } \ No newline at end of file diff --git a/src/project/flowr-project.ts b/src/project/flowr-project.ts index 9b037a1383c..e404e5a3539 100644 --- a/src/project/flowr-project.ts +++ b/src/project/flowr-project.ts @@ -20,7 +20,7 @@ export interface FlowrProject { export async function getDummyFlowrProject(){ const exampleFlowrProject: FlowrProject = { - analyzer: {} as FlowrAnalyzer, + analyzer: {} as FlowrAnalyzer, builder: {} as FlowrAnalyzerBuilder, plugins: [], projectRoot: path.resolve('test/testfiles/project/'), diff --git a/src/queries/catalog/config-query/config-query-executor.ts b/src/queries/catalog/config-query/config-query-executor.ts index f3692a0c1e3..a388cbbc4bc 100644 --- a/src/queries/catalog/config-query/config-query-executor.ts +++ b/src/queries/catalog/config-query/config-query-executor.ts @@ -14,13 +14,11 @@ export function executeConfigQuery({ input }: BasicQueryData, queries: readonly deepMergeObjectInPlace(input.flowrConfig, update); } - return new Promise((resolve) => { - resolve({ - '.meta': { - /* there is no sense in measuring a get */ - timing: 0 - }, - config: input.flowrConfig - }); + return Promise.resolve({ + '.meta': { + /* there is no sense in measuring a get */ + timing: 0 + }, + config: input.flowrConfig }); } diff --git a/src/queries/query.ts b/src/queries/query.ts index 3a0a62af5bc..cba9b5ae898 100644 --- a/src/queries/query.ts +++ b/src/queries/query.ts @@ -51,7 +51,6 @@ import type { FlowrConfigOptions } from '../config'; /** * These are all queries that can be executed from within flowR - * {@link SynchronousQuery} are queries that can be executed synchronously, i.e., they do not return a Promise. */ export type Query = CallContextQuery | ConfigQuery @@ -74,16 +73,12 @@ export type Query = CallContextQuery | LinterQuery ; - -export type SynchronousQuery = Exclude Promise }>; -export type SupportedSynchronousQueryTypes = SynchronousQuery['type']; - export type QueryArgumentsWithType = Query & { type: QueryType }; /* Each executor receives all queries of its type in case it wants to avoid repeated traversal */ export type QueryExecutor> = (data: BasicQueryData, query: readonly Query[]) => Result; -type SupportedQueries = { +type SupportedQueriesType = { [QueryType in Query['type']]: SupportedQuery } @@ -122,21 +117,18 @@ export const SupportedQueries = { 'project': ProjectQueryDefinition, 'origin': OriginQueryDefinition, 'linter': LinterQueryDefinition -} as const satisfies SupportedQueries; +} as const satisfies SupportedQueriesType; export type SupportedQueryTypes = keyof typeof SupportedQueries; -export type QueryResult = AsyncOrSync>; +export type QueryResult = Promise>; -export async function executeQueriesOfSameType(data: BasicQueryData, queries: readonly SpecificQuery[]): Promise> -export async function executeQueriesOfSameType(data: BasicQueryData, queries: readonly SpecificQuery[]): Promise> -export async function executeQueriesOfSameType(data: BasicQueryData, queries: readonly SpecificQuery[]): Promise> { +export async function executeQueriesOfSameType(data: BasicQueryData, queries: readonly SpecificQuery[]): QueryResult { guard(queries.length > 0, 'At least one query must be provided'); /* every query must have the same type */ guard(queries.every(q => q.type === queries[0].type), 'All queries must have the same type'); const query = SupportedQueries[queries[0].type]; guard(query !== undefined, `Unsupported query type: ${queries[0].type}`); - const result = await query.executor(data, queries as never); - return result as QueryResult; + return query.executor(data, queries as never) as QueryResult; } function isVirtualQuery< @@ -170,7 +162,7 @@ function groupQueriesByType< } /* a record mapping the query type present to its respective result */ -export type QueryResults = { +export type QueryResults = { readonly [QueryType in Base]: Awaited> } & BaseQueryResult @@ -194,14 +186,6 @@ function isPromiseLike(value: unknown): value is Promise { /** * This is the main query execution function that takes a set of queries and executes them on the given data. */ -export function executeQueries< - Base extends SupportedSynchronousQueryTypes, - VirtualArguments extends VirtualCompoundConstraint = VirtualCompoundConstraint ->(data: BasicQueryData, queries: Queries): QueryResults -export function executeQueries< - Base extends SupportedQueryTypes, - VirtualArguments extends VirtualCompoundConstraint = VirtualCompoundConstraint ->(data: BasicQueryData, queries: Queries): AsyncOrSync> export function executeQueries< Base extends SupportedQueryTypes, VirtualArguments extends VirtualCompoundConstraint = VirtualCompoundConstraint diff --git a/src/search/search-executor/search-enrichers.ts b/src/search/search-executor/search-enrichers.ts index 4eea7fe04f5..b0a8c7186c5 100644 --- a/src/search/search-executor/search-enrichers.ts +++ b/src/search/search-executor/search-enrichers.ts @@ -18,7 +18,7 @@ import { extractCfgQuick } from '../../control-flow/extract-cfg'; import { getOriginInDfg, OriginType } from '../../dataflow/origin/dfg-get-origin'; import { type NodeId, recoverName } from '../../r-bridge/lang-4.x/ast/model/processing/node-id'; import type { ControlFlowInformation } from '../../control-flow/control-flow-graph'; -import type { QueryResult, SynchronousQuery } from '../../queries/query'; +import type { Query, QueryResult } from '../../queries/query'; import type { CfgSimplificationPassName } from '../../control-flow/cfg-simplification'; import { cfgFindAllReachable, DefaultCfgSimplificationOrder } from '../../control-flow/cfg-simplification'; import type { AsyncOrSync, AsyncOrSyncType } from 'ts-essentials'; @@ -99,11 +99,11 @@ export interface CfgInformationArguments extends MergeableRecord { export interface QueryDataElementContent extends MergeableRecord { /** The name of the query that this element originated from. To get each query's data, see {@link QueryDataSearchContent}. */ - query: SynchronousQuery['type'] + query: Query['type'] } export interface QueryDataSearchContent extends MergeableRecord { - queries: { [QueryType in SynchronousQuery['type']]: AsyncOrSyncType> } + queries: { [QueryType in Query['type']]: Awaited> } } /** diff --git a/src/search/search-executor/search-generators.ts b/src/search/search-executor/search-generators.ts index 1f678d017b0..c80356398a4 100644 --- a/src/search/search-executor/search-generators.ts +++ b/src/search/search-executor/search-generators.ts @@ -5,7 +5,7 @@ import type { ParentInformation, RNodeWithParent } from '../../r-bridge/lang-4.x import type { SlicingCriteria } from '../../slicing/criterion/parse'; import { slicingCriterionToId } from '../../slicing/criterion/parse'; import { isNotUndefined } from '../../util/assert'; -import type { Query, SupportedQuery, SynchronousQuery } from '../../queries/query'; +import type { Query, SupportedQuery } from '../../queries/query'; import { executeQueries, SupportedQueries } from '../../queries/query'; import type { BaseQueryResult } from '../../queries/base-query-format'; import type { RNode } from '../../r-bridge/lang-4.x/ast/model/model'; @@ -88,7 +88,7 @@ function generateFrom(_input: FlowrAnalysisInput, args: { from: FlowrSearchEleme } async function generateFromQuery(input: FlowrAnalysisInput, args: { - from: readonly SynchronousQuery[] + from: readonly Query[] }): Promise[]>> { const result = await executeQueries({ input }, args.from); @@ -112,7 +112,9 @@ async function generateFromQuery(input: FlowrAnalysisInput, args: { const dataflow = await input.dataflow(); const cfg = await input.controlFlow(); - const elements = await new FlowrSearchElements([...nodesByQuery].flatMap(([_, nodes]) => [...nodes])).enrich(input, Enrichment.QueryData, { queries: result }); + const elements = await new FlowrSearchElements([...nodesByQuery] + .flatMap(([_, nodes]) => [...nodes])) + .enrich(input, Enrichment.QueryData, { queries: result }); return elements.mutate(s => Promise.all(s.map(async e => { const [query, _] = [...nodesByQuery].find(([_, nodes]) => nodes.has(e)) as [Query['type'], Set>]; return await enrichElement(e, elements, { normalize, dataflow, cfg }, Enrichment.QueryData, { query }); diff --git a/test/functionality/_helper/query.ts b/test/functionality/_helper/query.ts index 8f785243e29..aa54b9c2da4 100644 --- a/test/functionality/_helper/query.ts +++ b/test/functionality/_helper/query.ts @@ -1,4 +1,7 @@ -import type { DEFAULT_DATAFLOW_PIPELINE } from '../../../src/core/steps/pipeline/default-pipelines'; +import type { + DEFAULT_DATAFLOW_PIPELINE, + TREE_SITTER_DATAFLOW_PIPELINE +} from '../../../src/core/steps/pipeline/default-pipelines'; import { requestFromInput } from '../../../src/r-bridge/retriever'; import type { Query, QueryResults, QueryResultsWithoutMeta } from '../../../src/queries/query'; import { executeQueries, SupportedQueries } from '../../../src/queries/query'; @@ -8,17 +11,18 @@ import { decorateLabelContext } from './label'; import type { VirtualCompoundConstraint } from '../../../src/queries/virtual-query/compound-query'; import { log } from '../../../src/util/log'; import { dataflowGraphToMermaidUrl } from '../../../src/core/print/dataflow-printer'; -import type { PipelineOutput } from '../../../src/core/steps/pipeline/pipeline'; +import type { PipelineOutput, PipelinePerStepMetaInformation } from '../../../src/core/steps/pipeline/pipeline'; import { assert, test } from 'vitest'; import { cfgToMermaidUrl } from '../../../src/util/mermaid/cfg'; import { defaultConfigOptions } from '../../../src/config'; -import type { KnownParser } from '../../../src/r-bridge/parser'; +import type { KnownParser, ParseStepOutput } from '../../../src/r-bridge/parser'; import { extractCfg } from '../../../src/control-flow/extract-cfg'; import { getDummyFlowrProject } from '../../../src/project/flowr-project'; import { FlowrAnalyzerBuilder } from '../../../src/project/flowr-analyzer-builder'; +import type { Tree } from 'web-tree-sitter'; -function normalizeResults(result: Awaited>): QueryResultsWithoutMeta { +function normalizeResults(result: QueryResults): QueryResultsWithoutMeta { return JSON.parse(JSON.stringify(result, (key: unknown, value: unknown) => { if(key === '.meta') { return undefined; @@ -44,7 +48,7 @@ export function assertQuery< parser: KnownParser, code: string, queries: readonly (Queries | VirtualQueryArgumentsWithType)[], - expected: QueryResultsWithoutMeta | ((info: PipelineOutput) => (QueryResultsWithoutMeta | Promise>)) + expected: QueryResultsWithoutMeta | ((info: PipelineOutput) => (QueryResultsWithoutMeta | Promise>)) ) { const effectiveName = decorateLabelContext(name, ['query']); @@ -72,7 +76,10 @@ export function assertQuery< .build(); const dummyProject = await getDummyFlowrProject(); - const result = await Promise.resolve(executeQueries({ input: analyzer, libraries: dummyProject.libraries }, queries)); + const result = await executeQueries({ + input: analyzer, + libraries: dummyProject.libraries + }, queries); log.info(`total query time: ${result['.meta'].timing.toFixed(0)}ms (~1ms accuracy)`); @@ -83,7 +90,7 @@ export function assertQuery< // eslint-disable-next-line @typescript-eslint/no-unsafe-argument const expectedNormalized = normalizeResults(typeof expected === 'function' ? await expected( { - parse: await analyzer.parseOutput(), + parse: await analyzer.parseOutput() as ParseStepOutput & PipelinePerStepMetaInformation, normalize: await analyzer.normalizedAst(), dataflow: await analyzer.dataflow() } From 051f6599c4fc8b8bb66740156fd87682c18cb13b Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Thu, 25 Sep 2025 11:25:08 +0200 Subject: [PATCH 60/70] refactor: output tunes to existing tests --- test/functionality/project/package.test.ts | 5 +---- test/functionality/project/plugin/description-file.test.ts | 2 -- .../statistics/statistics-post-process.test.ts | 6 +++++- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/test/functionality/project/package.test.ts b/test/functionality/project/package.test.ts index b28f39c04a9..9ab2e1468aa 100644 --- a/test/functionality/project/package.test.ts +++ b/test/functionality/project/package.test.ts @@ -1,6 +1,6 @@ import { describe, test , assert } from 'vitest'; import { Package } from '../../../src/project/plugins/package-version-plugins/package'; -import { minVersion, Range } from 'semver'; +import { Range } from 'semver'; describe('DESCRIPTION-file', function() { describe.sequential('Parsing', function() { @@ -11,9 +11,6 @@ describe('DESCRIPTION-file', function() { p1.addInfo(undefined, undefined, undefined, new Range('>= 1.5')); p1.addInfo(undefined, undefined, undefined, new Range('<= 2.2.5')); - console.log(p1.derivedVersion); - console.log(minVersion(p1.derivedVersion as Range)); - assert.isTrue(p1.derivedVersion?.test('1.7.0')); }); }); diff --git a/test/functionality/project/plugin/description-file.test.ts b/test/functionality/project/plugin/description-file.test.ts index 38bf00e2eb6..bff7b685dc2 100644 --- a/test/functionality/project/plugin/description-file.test.ts +++ b/test/functionality/project/plugin/description-file.test.ts @@ -23,7 +23,6 @@ describe('DESCRIPTION-file', function() { await flowrAnalyzerPackageVersionsDescriptionFilePlugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); - console.log(flowrAnalyzerPackageVersionsDescriptionFilePlugin.packages); assert.isNotEmpty(flowrAnalyzerPackageVersionsDescriptionFilePlugin.packages); }); @@ -33,7 +32,6 @@ describe('DESCRIPTION-file', function() { await flowrAnalyzerLoadingOrderDescriptionFilePlugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); - console.log(flowrAnalyzerLoadingOrderDescriptionFilePlugin.loadingOrder); assert.isNotEmpty(flowrAnalyzerLoadingOrderDescriptionFilePlugin.loadingOrder); }); }); diff --git a/test/functionality/statistics/statistics-post-process.test.ts b/test/functionality/statistics/statistics-post-process.test.ts index 748bb9d30b3..85afc2f0579 100644 --- a/test/functionality/statistics/statistics-post-process.test.ts +++ b/test/functionality/statistics/statistics-post-process.test.ts @@ -5,13 +5,17 @@ import path from 'path'; import { flowrScriptSummarizer } from '../../../src/cli/script-core/summarizer-core'; import { SummarizerType } from '../../../src/util/summarizer'; import { getPlatform } from '../../../src/util/os'; -import { describe, test } from 'vitest'; +import { describe, test, vi } from 'vitest'; import { defaultConfigOptions } from '../../../src/config'; describe('Post-Processing', () => { /* if we are on windows, skip, as there are maybe cleanup problems */ test.skipIf(getPlatform() === 'windows' || getPlatform() === 'unknown')('Full Extraction on Sample Folder (Shellesc)', async() => { + // mock console.log to avoid cluttering the test output + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + const tempfolder = fs.mkdtempSync(path.resolve(os.tmpdir(), 'flowr-test-temp-')); // run the basic statistics script await flowrScriptGetStats({ From 44e50cc2d8b7f47c34dbb1b52b22a8ded07009d5 Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Thu, 25 Sep 2025 11:35:14 +0200 Subject: [PATCH 61/70] refactor: minor patches to asyncness --- src/cli/repl/server/connection.ts | 8 +++++--- src/linter/rules/file-path-validity.ts | 4 ++-- .../dependencies-query/dependencies-query-format.ts | 5 +++-- src/search/search-executor/search-enrichers.ts | 4 ++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index acf2cb6ccc9..15f8d450413 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -28,6 +28,7 @@ import { PARSE_WITH_R_SHELL_STEP } from '../../../core/steps/all/core/00-parse'; import { NORMALIZE } from '../../../core/steps/all/core/10-normalize'; import { STATIC_DATAFLOW } from '../../../core/steps/all/core/20-dataflow'; import { ansiFormatter, voidFormatter } from '../../../util/text/ansi'; +import type { TREE_SITTER_DATAFLOW_PIPELINE } from '../../../core/steps/pipeline/default-pipelines'; import { DEFAULT_SLICING_PIPELINE } from '../../../core/steps/pipeline/default-pipelines'; import type { PipelineOutput } from '../../../core/steps/pipeline/pipeline'; import type { DeepPartial } from 'ts-essentials'; @@ -50,6 +51,7 @@ import type { FlowrAnalyzer } from '../../../project/flowr-analyzer'; import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { DataflowInformation } from '../../../dataflow/info'; import { PipelineStepStage } from '../../../core/steps/pipeline-step'; +import type { Tree } from 'web-tree-sitter'; /** * Each connection handles a single client, answering to its requests. @@ -113,7 +115,7 @@ export class FlowRServerConnection { this.handleRepl(request.message as ExecuteRequestMessage); break; case 'request-lineage': - this.handleLineageRequest(request.message as LineageRequestMessage); + void this.handleLineageRequest(request.message as LineageRequestMessage); break; case 'request-query': this.handleQueryRequest(request.message as QueryRequestMessage); @@ -384,9 +386,9 @@ export class FlowRServerConnection { } } -export function sanitizeAnalysisResults(parse: ParseStepOutput, normalize: NormalizedAst, dataflow: DataflowInformation): DeepPartial> { +export function sanitizeAnalysisResults(parse: ParseStepOutput, normalize: NormalizedAst, dataflow: DataflowInformation): DeepPartial> { return { - parse, + parse: parse as ParseStepOutput, normalize: { ...normalize, idMap: undefined diff --git a/src/linter/rules/file-path-validity.ts b/src/linter/rules/file-path-validity.ts index 66049082903..fc9d26fa7f1 100644 --- a/src/linter/rules/file-path-validity.ts +++ b/src/linter/rules/file-path-validity.ts @@ -51,7 +51,7 @@ export const FILE_PATH_VALIDITY = { readFunctions: config.additionalReadFunctions, writeFunctions: config.additionalWriteFunctions }).with(Enrichment.CfgInformation), - processSearchResult: async(elements, config, data): Promise<{ results: FilePathValidityResult[], '.meta': FilePathValidityMetadata }> => { + processSearchResult: (elements, config, data): { results: FilePathValidityResult[], '.meta': FilePathValidityMetadata } => { const cfg = elements.enrichmentContent(Enrichment.CfgInformation).cfg.graph; const metadata: FilePathValidityMetadata = { totalReads: 0, @@ -59,7 +59,7 @@ export const FILE_PATH_VALIDITY = { totalWritesBeforeAlways: 0, totalValid: 0 }; - const results = await elements.enrichmentContent(Enrichment.QueryData).queries['dependencies']; + const results = elements.enrichmentContent(Enrichment.QueryData).queries['dependencies']; return { results: elements.getElements().flatMap(element => { const matchingRead = results.read.find(r => r.nodeId == element.node.info.id); diff --git a/src/queries/catalog/dependencies-query/dependencies-query-format.ts b/src/queries/catalog/dependencies-query/dependencies-query-format.ts index cde4a3bdb48..89fa7f06a16 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-format.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-format.ts @@ -15,6 +15,7 @@ import { visitAst } from '../../../r-bridge/lang-4.x/ast/model/processing/visito import { RType } from '../../../r-bridge/lang-4.x/ast/model/type'; import type { CallContextQueryResult } from '../call-context-query/call-context-query-format'; import type { Range } from 'semver'; +import type { AsyncOrSync } from 'ts-essentials'; export const Unknown = 'unknown'; @@ -22,7 +23,7 @@ export interface DependencyCategorySettings { queryDisplayName?: string functions: FunctionInfo[] defaultValue?: string - additionalAnalysis?: (data: BasicQueryData, ignoreDefault: boolean, functions: FunctionInfo[], queryResults: CallContextQueryResult, result: DependencyInfo[]) => void + additionalAnalysis?: (data: BasicQueryData, ignoreDefault: boolean, functions: FunctionInfo[], queryResults: CallContextQueryResult, result: DependencyInfo[]) => AsyncOrSync } export const DefaultDependencyCategories = { @@ -109,7 +110,7 @@ function printResultSection(title: string, infos: DependencyInfo[], result: stri }, new Map()); for(const [functionName, infos] of grouped) { result.push(` â•° \`${functionName}\``); - result.push(infos.map(i => ` â•° Node Id: ${i.nodeId}${i.value !== undefined ? `, \`${i.value}\`` : ''}${i.derivedVersion !== undefined ? `, Version: \`${i.derivedVersion.toString()}\`` : ''}`).join('\n')); + result.push(infos.map(i => ` â•° Node Id: ${i.nodeId}${i.value !== undefined ? `, \`${i.value}\`` : ''}${i.derivedVersion !== undefined ? `, Version: \`${i.derivedVersion.format()}\`` : ''}`).join('\n')); } } diff --git a/src/search/search-executor/search-enrichers.ts b/src/search/search-executor/search-enrichers.ts index b0a8c7186c5..085c4cc50d3 100644 --- a/src/search/search-executor/search-enrichers.ts +++ b/src/search/search-executor/search-enrichers.ts @@ -178,8 +178,8 @@ export const Enrichments = { mapper: ({ linkedIds }) => linkedIds } satisfies EnrichmentData[]>, [Enrichment.CfgInformation]: { - enrichElement: async(e, search, _data, _args, prev) => { - const searchContent: CfgInformationSearchContent = await search.enrichmentContent(Enrichment.CfgInformation); + enrichElement: (e, search, _data, _args, prev) => { + const searchContent: CfgInformationSearchContent = search.enrichmentContent(Enrichment.CfgInformation); return { ...prev, isRoot: searchContent.cfg.graph.rootIds().has(e.node.info.id), From d69e1c2fff924115a6e6e1a1f29779b9d0b2a466 Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Thu, 25 Sep 2025 23:22:40 +0200 Subject: [PATCH 62/70] refactor: design of separate cache api --- src/cli/repl/commands/repl-cfg.ts | 6 +- src/cli/repl/commands/repl-normalize.ts | 4 +- src/cli/repl/commands/repl-parse.ts | 2 +- src/cli/repl/commands/repl-query.ts | 2 +- src/cli/repl/server/connection.ts | 4 +- src/core/steps/pipeline/pipeline.ts | 10 -- src/linter/linter-executor.ts | 4 +- src/project/cache/flowr-analyzer-cache.ts | 48 ++++++++++ src/project/cache/flowr-cache.ts | 60 ++++++++++++ src/project/cache/flowr-controlflow-cache.ts | 36 +++++++ src/project/cache/flowr-dataflow-cache.ts | 21 +++++ .../cache/flowr-normalized-ast-cache.ts | 22 +++++ src/project/cache/flowr-parse-cache.ts | 26 ++++++ src/project/flowr-analyzer.ts | 93 ++++++++++--------- .../call-context-query-executor.ts | 4 +- .../control-flow-query-executor.ts | 2 +- .../dependencies-query-executor.ts | 2 +- .../dependencies-query-format.ts | 2 +- .../df-shape-query/df-shape-query-executor.ts | 4 +- .../happens-before-query-executor.ts | 2 +- .../id-map-query/id-map-query-executor.ts | 2 +- .../lineage-query/lineage-query-executor.ts | 2 +- .../location-map-query-executor.ts | 2 +- .../normalized-ast-query-executor.ts | 2 +- .../origin-query/origin-query-executor.ts | 2 +- .../resolve-value-query-executor.ts | 2 +- .../static-slice-query-executor.ts | 4 +- .../search-executor/search-enrichers.ts | 2 +- .../search-executor/search-generators.ts | 12 +-- .../search-executor/search-transformer.ts | 4 +- test/functionality/_helper/query.ts | 2 +- test/functionality/cli/server.test.ts | 4 +- 32 files changed, 302 insertions(+), 92 deletions(-) create mode 100644 src/project/cache/flowr-analyzer-cache.ts create mode 100644 src/project/cache/flowr-cache.ts create mode 100644 src/project/cache/flowr-controlflow-cache.ts create mode 100644 src/project/cache/flowr-dataflow-cache.ts create mode 100644 src/project/cache/flowr-normalized-ast-cache.ts create mode 100644 src/project/cache/flowr-parse-cache.ts diff --git a/src/cli/repl/commands/repl-cfg.ts b/src/cli/repl/commands/repl-cfg.ts index 94a1b915748..406a7d565dd 100644 --- a/src/cli/repl/commands/repl-cfg.ts +++ b/src/cli/repl/commands/repl-cfg.ts @@ -14,9 +14,9 @@ function formatInfo(out: ReplOutput, type: string): string { } async function produceAndPrintCfg(analyzer: FlowrAnalysisInput, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string) { - const cfg = await analyzer.controlFlow([...DefaultCfgSimplificationOrder, ...simplifications]); - const result = await analyzer.normalizedAst(); - const mermaid = cfgConverter(cfg, result); + const cfg = await analyzer.controlflow([...DefaultCfgSimplificationOrder, ...simplifications]); + const normalizedAst = await analyzer.normalize(); + const mermaid = cfgConverter(cfg, normalizedAst); output.stdout(mermaid); try { const clipboard = await import('clipboardy'); diff --git a/src/cli/repl/commands/repl-normalize.ts b/src/cli/repl/commands/repl-normalize.ts index fff1bdf0a03..6649b43fe9e 100644 --- a/src/cli/repl/commands/repl-normalize.ts +++ b/src/cli/repl/commands/repl-normalize.ts @@ -17,7 +17,7 @@ export const normalizeCommand: ReplCodeCommand = { script: false, argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { - const result = await analyzer.normalizedAst(); + const result = await analyzer.normalize(); const mermaid = normalizedAstToMermaid(result.ast); output.stdout(mermaid); try { @@ -36,7 +36,7 @@ export const normalizeStarCommand: ReplCodeCommand = { script: false, argsParser: (args: string) => handleString(args), fn: async({ output, analyzer }) => { - const result = await analyzer.normalizedAst(); + const result = await analyzer.normalize(); const mermaid = normalizedAstToMermaidUrl(result.ast); output.stdout(mermaid); try { diff --git a/src/cli/repl/commands/repl-parse.ts b/src/cli/repl/commands/repl-parse.ts index 4e90ecc922f..661008ddc19 100644 --- a/src/cli/repl/commands/repl-parse.ts +++ b/src/cli/repl/commands/repl-parse.ts @@ -166,7 +166,7 @@ export const parseCommand: ReplCodeCommand = { }; }, fn: async({ output, analyzer }) => { - const result = await analyzer.parseOutput(); + const result = await analyzer.parse(); if(analyzer.parserName() === 'r-shell') { const object = convertPreparedParsedData(prepareParsedData(result.parsed as unknown as string)); diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index ff9c87b28b9..547c8931546 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -72,7 +72,7 @@ async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisInput libraries: dummyProject.libraries }, parsedQuery)), parsedQuery, - processed: { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalizedAst() } + processed: { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalize() } }; } diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index 15f8d450413..bc93530394e 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -172,7 +172,7 @@ export class FlowRServerConnection { } const config = (): QuadSerializationConfiguration => ({ context: message.filename ?? 'unknown', getId: defaultQuadIdGenerator() }); - const sanitizedResults = sanitizeAnalysisResults(await analyzer.parseOutput(), await analyzer.normalizedAst(), await analyzer.dataflow()); + const sanitizedResults = sanitizeAnalysisResults(await analyzer.parse(), await analyzer.normalizedAst(), await analyzer.dataflow()); if(message.format === 'n-quads') { sendMessage(this.socket, { type: 'response-file-analysis', @@ -180,7 +180,7 @@ export class FlowRServerConnection { id: message.id, cfg: cfg ? cfg2quads(cfg, config()) : undefined, results: { - parse: await printStepResult(PARSE_WITH_R_SHELL_STEP, await analyzer.parseOutput() as ParseStepOutput, StepOutputFormat.RdfQuads, config()), + parse: await printStepResult(PARSE_WITH_R_SHELL_STEP, await analyzer.parse() as ParseStepOutput, StepOutputFormat.RdfQuads, config()), normalize: await printStepResult(NORMALIZE, await analyzer.normalizedAst(), StepOutputFormat.RdfQuads, config()), dataflow: await printStepResult(STATIC_DATAFLOW, await analyzer.dataflow(), StepOutputFormat.RdfQuads, config()) } diff --git a/src/core/steps/pipeline/pipeline.ts b/src/core/steps/pipeline/pipeline.ts index 2705cc3a2e7..af940ba8ab5 100644 --- a/src/core/steps/pipeline/pipeline.ts +++ b/src/core/steps/pipeline/pipeline.ts @@ -47,16 +47,6 @@ export interface PipelinePerStepMetaInformation { * the required accuracy is dependent on the measuring system, but usually at around 1 ms. */ readonly timing: number - /** - * The result was newly created - */ - readonly cached: false; - } | { - /** - * The result was cached. - * Thus, no timing information is available. - */ - readonly cached: true; } } diff --git a/src/linter/linter-executor.ts b/src/linter/linter-executor.ts index 1e84de69379..23150fb4721 100644 --- a/src/linter/linter-executor.ts +++ b/src/linter/linter-executor.ts @@ -20,9 +20,9 @@ export async function executeLintingRule(ruleName const processStart = Date.now(); const result = await rule.processSearchResult(searchResult, fullConfig, { - normalize: await input.normalizedAst(), + normalize: await input.normalize(), dataflow: await input.dataflow(), - cfg: await input.controlFlow(), + cfg: await input.controlflow(), config: input.flowrConfig, } ); diff --git a/src/project/cache/flowr-analyzer-cache.ts b/src/project/cache/flowr-analyzer-cache.ts new file mode 100644 index 00000000000..6cf2b11a841 --- /dev/null +++ b/src/project/cache/flowr-analyzer-cache.ts @@ -0,0 +1,48 @@ +import type { KnownParser } from '../../r-bridge/parser'; +import { FlowrParseCache } from './flowr-parse-cache'; +import { FlowrNormalizeCache } from './flowr-normalized-ast-cache'; +import { FlowrDataflowCache } from './flowr-dataflow-cache'; +import type { CacheInvalidationEvent, CacheInvalidationEventReceiver } from './flowr-cache'; +import { CacheInvalidationEventType } from './flowr-cache'; +import { FlowrControlflowCache } from './flowr-controlflow-cache'; +import type { DEFAULT_DATAFLOW_PIPELINE, TREE_SITTER_DATAFLOW_PIPELINE } from '../../core/steps/pipeline/default-pipelines'; + +export interface FlowrAnalyzerCaches { + parse: FlowrParseCache; + normalize: FlowrNormalizeCache; + dataflow: FlowrDataflowCache; + controlflow: FlowrControlflowCache; +} + + +/** + * This provides the full analyzer caching layer + */ +export class FlowrAnalyzerCache implements CacheInvalidationEventReceiver, FlowrAnalyzerCaches { + public readonly parse: FlowrParseCache; + public readonly normalize: FlowrNormalizeCache; + public readonly dataflow: FlowrDataflowCache; + public readonly controlflow: FlowrControlflowCache; + + /** the currently running pipeline */ + private pipeline: typeof DEFAULT_DATAFLOW_PIPELINE | typeof TREE_SITTER_DATAFLOW_PIPELINE; + + protected constructor(parser: Parser) { + this.parse = FlowrParseCache.create(parser); + this.normalize = FlowrNormalizeCache.create(this.parse); + this.dataflow = FlowrDataflowCache.create(this.normalize); + this.controlflow = FlowrControlflowCache.create(this.dataflow); + } + + public static create(parser: Parser): FlowrAnalyzerCache { + return new FlowrAnalyzerCache(parser); + } + + public receive(event: CacheInvalidationEvent) { + this.parse.receive(event); + } + + public reset() { + this.receive({ type: CacheInvalidationEventType.Full }); + } +} \ No newline at end of file diff --git a/src/project/cache/flowr-cache.ts b/src/project/cache/flowr-cache.ts new file mode 100644 index 00000000000..eacb06c4f0d --- /dev/null +++ b/src/project/cache/flowr-cache.ts @@ -0,0 +1,60 @@ +import { assertUnreachable } from '../../util/assert'; + +export const enum CacheInvalidationEventType { + Full = 'full' +} +export type CacheInvalidationEvent = + { type: CacheInvalidationEventType.Full } + +export interface CacheInvalidationEventReceiver { + receive(event: CacheInvalidationEvent): void +} + +/** + * Central class for caching analysis results in FlowR. + */ +export abstract class FlowrCache implements CacheInvalidationEventReceiver { + private value: Cache | undefined = undefined; + private dependents: CacheInvalidationEventReceiver[] = []; + + public registerDependent(dependent: CacheInvalidationEventReceiver) { + this.dependents.push(dependent); + } + public removeDependent(dependent: CacheInvalidationEventReceiver) { + this.dependents = this.dependents.filter(d => d !== dependent); + } + + receive(event: CacheInvalidationEvent): void { + /* we will update this as soon as we support incremental update patterns */ + switch(event.type) { + case CacheInvalidationEventType.Full: + this.value = undefined; + break; + default: + assertUnreachable(event.type); + } + /* in the future we want to defer this *after* the dataflow is re-computed, then all receivers can decide whether they need to update */ + this.notifyDependents(event); + } + + /** + * Notify all dependents of a cache invalidation event. + */ + public notifyDependents(event: CacheInvalidationEvent) { + for(const dependent of this.dependents) { + dependent.receive(event); + } + } + + /** + * Get the cached value or compute it if not present. + * This will, by default, not trigger any {@link notifyDependents} calls, as this is only a cache retrieval. + */ + public computeIfAbsent(force: boolean | undefined, compute: () => Cache): Cache { + if(this.value === undefined || force) { + this.value = compute(); + } + return this.value; + } + +} \ No newline at end of file diff --git a/src/project/cache/flowr-controlflow-cache.ts b/src/project/cache/flowr-controlflow-cache.ts new file mode 100644 index 00000000000..0c02863842e --- /dev/null +++ b/src/project/cache/flowr-controlflow-cache.ts @@ -0,0 +1,36 @@ +import { FlowrCache } from './flowr-cache'; +import type { KnownParser } from '../../r-bridge/parser'; +import type { ObjectMap } from '../../util/collections/objectmap'; +import type { CfgSimplificationPassName } from '../../control-flow/cfg-simplification'; +import type { ControlFlowInformation } from '../../control-flow/control-flow-graph'; +import type { FlowrDataflowCache } from './flowr-dataflow-cache'; + + +interface ControlFlowCache { + simplified: ObjectMap, + quick: ControlFlowInformation +} + +/** + * Controlflow specific cache. + */ +export class FlowrControlflowCache extends FlowrCache< + ControlFlowCache +> { + + protected constructor(dataflow: FlowrDataflowCache) { + super(); + dataflow.registerDependent(this); + } + + public static create(dataflow: FlowrDataflowCache): FlowrControlflowCache { + return new FlowrControlflowCache(dataflow); + } + // TODO: updates + /* + this.controlFlowInfos = { + simplified: new ObjectMap(), + quick: undefined as unknown as ControlFlowInformation + }; + */ +} \ No newline at end of file diff --git a/src/project/cache/flowr-dataflow-cache.ts b/src/project/cache/flowr-dataflow-cache.ts new file mode 100644 index 00000000000..62aec446a3c --- /dev/null +++ b/src/project/cache/flowr-dataflow-cache.ts @@ -0,0 +1,21 @@ +import { FlowrCache } from './flowr-cache'; +import type { KnownParser } from '../../r-bridge/parser'; +import type { DataflowInformation } from '../../dataflow/info'; +import type { FlowrNormalizeCache } from './flowr-normalized-ast-cache'; + +/** + * Dataflow specific cache. + */ +export class FlowrDataflowCache extends FlowrCache< + DataflowInformation +> { + + protected constructor(normalizer: FlowrNormalizeCache) { + super(); + normalizer.registerDependent(this); + } + + public static create(normalizer: FlowrNormalizeCache): FlowrDataflowCache { + return new FlowrDataflowCache(normalizer); + } +} \ No newline at end of file diff --git a/src/project/cache/flowr-normalized-ast-cache.ts b/src/project/cache/flowr-normalized-ast-cache.ts new file mode 100644 index 00000000000..8eeabbec89b --- /dev/null +++ b/src/project/cache/flowr-normalized-ast-cache.ts @@ -0,0 +1,22 @@ +import { FlowrCache } from './flowr-cache'; +import type { NormalizedAst } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { FlowrParseCache } from './flowr-parse-cache'; +import type { KnownParser } from '../../r-bridge/parser'; + +/** + * Flowr normalized AST specific cache. + */ +export class FlowrNormalizeCache extends FlowrCache< + NormalizedAst +> { + + protected constructor(parser: FlowrParseCache) { + super(); + parser.registerDependent(this); + } + + public static create(parser: FlowrParseCache): FlowrNormalizeCache { + /* in the future, with e.g. incrementality, we want to specialize based on the parser */ + return new FlowrNormalizeCache(parser); + } +} \ No newline at end of file diff --git a/src/project/cache/flowr-parse-cache.ts b/src/project/cache/flowr-parse-cache.ts new file mode 100644 index 00000000000..6b3bd3a065a --- /dev/null +++ b/src/project/cache/flowr-parse-cache.ts @@ -0,0 +1,26 @@ +import { FlowrCache } from './flowr-cache'; +import type { KnownParser, ParseStepOutput } from '../../r-bridge/parser'; +import type { PipelinePerStepMetaInformation } from '../../core/steps/pipeline/pipeline'; + +/** + * Flowr Parser specific cache. + */ +export class FlowrParseCache extends FlowrCache< + ParseStepOutput>> & PipelinePerStepMetaInformation +> { + private readonly parser: Parser; + + protected constructor(parser: Parser) { + super(); + this.parser = parser; + } + + public static create(parser: Parser): FlowrParseCache { + /* in the future, with e.g. incrementality, we want to specialize based on the parser */ + return new FlowrParseCache(parser); + } + + public getParser(): Parser { + return this.parser; + } +} \ No newline at end of file diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index 9a032f8f555..b48d4e033d5 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -1,11 +1,13 @@ import type { FlowrConfigOptions } from '../config'; import type { RParseRequests } from '../r-bridge/retriever'; +import type { DEFAULT_DATAFLOW_PIPELINE +} from '../core/steps/pipeline/default-pipelines'; import { createDataflowPipeline, createNormalizePipeline, createParsePipeline } from '../core/steps/pipeline/default-pipelines'; -import type { KnownParser, ParseStepOutput } from '../r-bridge/parser'; +import type { KnownParser, KnownParserName, ParseStepOutput } from '../r-bridge/parser'; import type { Queries, QueryResults, SupportedQueryTypes } from '../queries/query'; import { executeQueries } from '../queries/query'; import { extractCfg, extractCfgQuick } from '../control-flow/extract-cfg'; @@ -13,44 +15,39 @@ import type { ControlFlowInformation } from '../control-flow/control-flow-graph' import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { DataflowInformation } from '../dataflow/info'; import type { CfgSimplificationPassName } from '../control-flow/cfg-simplification'; -import type { PipelinePerStepMetaInformation } from '../core/steps/pipeline/pipeline'; +import type { PipelineInput, PipelinePerStepMetaInformation } from '../core/steps/pipeline/pipeline'; import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize'; -import { ObjectMap } from '../util/collections/objectmap'; +import { FlowrAnalyzerCache } from './cache/flowr-analyzer-cache'; /** - * Exposes the central analyses and information provided by the {@link FlowrAnalyzer} to the linter, search, and query APIs + * Exposes the central analyses and information provided by the {@link FlowrAnalyzer} to the linter, search, and query APIs. + * This allows us to exchange the underlying implementation of the analyzer without affecting the APIs. */ export type FlowrAnalysisInput = { parserName(): string - parseOutput(force?: boolean): Promise>> & PipelinePerStepMetaInformation> - normalizedAst(force?: boolean): Promise; + parse(force?: boolean): Promise>> & PipelinePerStepMetaInformation> + normalize(force?: boolean): Promise; dataflow(force?: boolean): Promise; - controlFlow(simplifications?: readonly CfgSimplificationPassName[], useDataflow?: boolean, force?: boolean): Promise; + controlflow(simplifications?: readonly CfgSimplificationPassName[], useDataflow?: boolean, force?: boolean): Promise; flowrConfig: FlowrConfigOptions; } -interface ControlFlowCache { - simplified: ObjectMap, - quick: ControlFlowInformation -} + /** * Central class for creating analyses in FlowR. * Use the {@link FlowrAnalyzerBuilder} to create a new instance. */ export class FlowrAnalyzer { + /** This is the config used for the analyzer */ public readonly flowrConfig: FlowrConfigOptions; + /** The currently active request that governs this analyzer */ private readonly request: RParseRequests; + /** The parser and engine backend */ private readonly parser: Parser; - private readonly requiredInput: Omit; + private readonly requiredInput: Omit, 'parser' | 'request'>; - private parse = undefined as unknown as ParseStepOutput>> & PipelinePerStepMetaInformation; - private ast = undefined as unknown as NormalizedAst; - private dataflowInfo = undefined as unknown as DataflowInformation; - private controlFlowInfos: ControlFlowCache = { - simplified: new ObjectMap(), - quick: undefined as unknown as ControlFlowInformation - }; + private readonly cache: FlowrAnalyzerCache; /** * Create a new analyzer instance. @@ -65,30 +62,40 @@ export class FlowrAnalyzer { this.request = request; this.parser = parser; this.requiredInput = requiredInput; + this.cache = FlowrAnalyzerCache.create(parser); } + /** + * Reset all caches used by the analyzer and effectively force all analyses to be redone. + */ public reset() { - this.ast = undefined as unknown as NormalizedAst; - this.dataflowInfo = undefined as unknown as DataflowInformation; - this.controlFlowInfos = { - simplified: new ObjectMap(), - quick: undefined as unknown as ControlFlowInformation - }; + this.cache.reset(); } - public parserName(): string { + /** + * Get the name of the parser used by the analyzer. + */ + public parserName(): KnownParserName { return this.parser.name; } /** * Get the parse output for the request. + * * The parse result type depends on the {@link KnownParser} used by the analyzer. * @param force - Do not use the cache, instead force a new parse. */ - public async parseOutput(force?: boolean): Promise>> & PipelinePerStepMetaInformation> { - if(this.parse && !force) { + public async parse(force?: boolean): Promise>> & PipelinePerStepMetaInformation> { + + return this.cache.parse.computeIfAbsent(force, + () => { + + } + ); + + if(this.parseCache && !force) { return { - ...this.parse, + ...this.parseCache, '.meta': { cached: true } @@ -99,8 +106,8 @@ export class FlowrAnalyzer { this.parser, { request: this.request }, this.flowrConfig).allRemainingSteps(); - this.parse = result.parse as unknown as ParseStepOutput>> & PipelinePerStepMetaInformation; - return this.parse; + this.parseCache = result.parse as unknown as ParseStepOutput>> & PipelinePerStepMetaInformation; + return this.parseCache; } /** @@ -108,9 +115,9 @@ export class FlowrAnalyzer { * @param force - Do not use the cache, instead force new analyses. */ public async normalizedAst(force?: boolean): Promise { - if(this.ast && !force) { + if(this.normalizeCache && !force) { return { - ...this.ast, + ...this.normalizeCache, '.meta': { cached: true } @@ -121,7 +128,7 @@ export class FlowrAnalyzer { this.parser, { request: this.request, ...this.requiredInput }, this.flowrConfig).allRemainingSteps(); - this.ast = result.normalize; + this.normalizeCache = result.normalize; return result.normalize; } @@ -130,9 +137,9 @@ export class FlowrAnalyzer { * @param force - Do not use the cache, instead force new analyses. */ public async dataflow(force?: boolean): Promise { - if(this.dataflowInfo && !force) { + if(this.dataflowCache && !force) { return { - ...this.dataflowInfo, + ...this.dataflowCache, '.meta': { cached: true } @@ -143,8 +150,8 @@ export class FlowrAnalyzer { this.parser, { request: this.request }, this.flowrConfig).allRemainingSteps(); - this.dataflowInfo = result.dataflow; - this.ast = result.normalize; + this.dataflowCache = result.dataflow; + this.normalizeCache = result.normalize; return result.dataflow; } @@ -162,15 +169,15 @@ export class FlowrAnalyzer { } } - if(force || !this.ast) { + if(force || !this.normalizeCache) { await this.normalizedAst(force); } - if(useDataflow && (force || !this.dataflowInfo)) { + if(useDataflow && (force || !this.dataflowCache)) { await this.dataflow(force); } - const result = extractCfg(this.ast, this.flowrConfig, this.dataflowInfo?.graph, simplifications); + const result = extractCfg(this.normalizeCache, this.flowrConfig, this.dataflowCache?.graph, simplifications); this.controlFlowInfos.simplified.set(simplifications ?? [], result); return result; } @@ -192,11 +199,11 @@ export class FlowrAnalyzer { } } - if(force || !this.ast) { + if(force || !this.normalizeCache) { await this.normalizedAst(force); } - const result = extractCfgQuick(this.ast); + const result = extractCfgQuick(this.normalizeCache); this.controlFlowInfos.quick = result; return result; } diff --git a/src/queries/catalog/call-context-query/call-context-query-executor.ts b/src/queries/catalog/call-context-query/call-context-query-executor.ts index 708fdba742e..877429e8ab0 100644 --- a/src/queries/catalog/call-context-query/call-context-query-executor.ts +++ b/src/queries/catalog/call-context-query/call-context-query-executor.ts @@ -213,7 +213,7 @@ function isParameterDefaultValue(nodeId: NodeId, ast: NormalizedAst): boolean { * 4. Attach `linkTo` calls to the respective calls. */ export async function executeCallContextQueries({ input }: BasicQueryData, queries: readonly CallContextQuery[]): Promise { - const ast = await input.normalizedAst(); + const ast = await input.normalize(); const dataflow = await input.dataflow(); /* omit performance page load */ @@ -226,7 +226,7 @@ export async function executeCallContextQueries({ input }: BasicQueryData, queri let cfg = undefined; if(requiresCfg) { - cfg = await input.controlFlow([], true); + cfg = await input.controlflow([], true); } const queriesWhichWantAliases = promotedQueries.filter(q => q.includeAliases); diff --git a/src/queries/catalog/control-flow-query/control-flow-query-executor.ts b/src/queries/catalog/control-flow-query/control-flow-query-executor.ts index 8f321abf972..1cd40ad27d1 100644 --- a/src/queries/catalog/control-flow-query/control-flow-query-executor.ts +++ b/src/queries/catalog/control-flow-query/control-flow-query-executor.ts @@ -11,7 +11,7 @@ export async function executeControlFlowQuery({ input }: BasicQueryData, queries const query = queries[0]; const start = Date.now(); - const controlFlow = await input.controlFlow(query.config?.simplificationPasses, true); + const controlFlow = await input.controlflow(query.config?.simplificationPasses, true); return { '.meta': { timing: Date.now() - start diff --git a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts index ae30914d008..9ead69ccc2e 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts @@ -38,7 +38,7 @@ export async function executeDependenciesQuery({ libraries: libraries }; - const normalize = await input.normalizedAst(); + const normalize = await input.normalize(); const dataflow = await input.dataflow(); const config = input.flowrConfig; diff --git a/src/queries/catalog/dependencies-query/dependencies-query-format.ts b/src/queries/catalog/dependencies-query/dependencies-query-format.ts index 89fa7f06a16..09127b5aad3 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-format.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-format.ts @@ -34,7 +34,7 @@ export const DefaultDependencyCategories = { /* for libraries, we have to additionally track all uses of `::` and `:::`, for this we currently simply traverse all uses */ additionalAnalysis: async(data, ignoreDefault, _functions, _queryResults, result) => { if(!ignoreDefault) { - visitAst((await data.input.normalizedAst()).ast, n => { + visitAst((await data.input.normalize()).ast, n => { if(n.type === RType.Symbol && n.namespace) { /* we should improve the identification of ':::' */ result.push({ diff --git a/src/queries/catalog/df-shape-query/df-shape-query-executor.ts b/src/queries/catalog/df-shape-query/df-shape-query-executor.ts index 893ac32c903..4b52f646963 100644 --- a/src/queries/catalog/df-shape-query/df-shape-query-executor.ts +++ b/src/queries/catalog/df-shape-query/df-shape-query-executor.ts @@ -15,12 +15,12 @@ export async function executeDfShapeQuery({ input }: BasicQueryData, queries: re queries = [{ type: 'df-shape' }]; } - const ast = await input.normalizedAst(); + const ast = await input.normalize(); const graph = (await input.dataflow()).graph; const start = Date.now(); const domains = inferDataFrameShapes( - await input.controlFlow(), + await input.controlflow(), graph, ast, input.flowrConfig); diff --git a/src/queries/catalog/happens-before-query/happens-before-query-executor.ts b/src/queries/catalog/happens-before-query/happens-before-query-executor.ts index 0652659fa00..00020f7a2f6 100644 --- a/src/queries/catalog/happens-before-query/happens-before-query-executor.ts +++ b/src/queries/catalog/happens-before-query/happens-before-query-executor.ts @@ -9,7 +9,7 @@ import { slicingCriterionToId } from '../../../slicing/criterion/parse'; export async function executeHappensBefore({ input }: BasicQueryData, queries: readonly HappensBeforeQuery[]): Promise { const start = Date.now(); const results: Record = {}; - const ast = await input.normalizedAst(); + const ast = await input.normalize(); const cfg = extractCfgQuick(ast); for(const query of queries) { const { a, b } = query; diff --git a/src/queries/catalog/id-map-query/id-map-query-executor.ts b/src/queries/catalog/id-map-query/id-map-query-executor.ts index 585c99bc4d7..c44d1969285 100644 --- a/src/queries/catalog/id-map-query/id-map-query-executor.ts +++ b/src/queries/catalog/id-map-query/id-map-query-executor.ts @@ -12,6 +12,6 @@ export async function executeIdMapQuery({ input }: BasicQueryData, queries: read /* there is no sense in measuring a get */ timing: 0 }, - idMap: (await input.normalizedAst()).idMap + idMap: (await input.normalize()).idMap }; } diff --git a/src/queries/catalog/lineage-query/lineage-query-executor.ts b/src/queries/catalog/lineage-query/lineage-query-executor.ts index e914dfeeb33..b7fcf32b513 100644 --- a/src/queries/catalog/lineage-query/lineage-query-executor.ts +++ b/src/queries/catalog/lineage-query/lineage-query-executor.ts @@ -10,7 +10,7 @@ export async function executeLineageQuery({ input }: BasicQueryData, queries: re if(result[criterion]) { log.warn('Duplicate criterion in lineage query:', criterion); } - result[criterion] = getLineage(criterion, (await input.dataflow()).graph, (await input.normalizedAst()).idMap); + result[criterion] = getLineage(criterion, (await input.dataflow()).graph, (await input.normalize()).idMap); } return { diff --git a/src/queries/catalog/location-map-query/location-map-query-executor.ts b/src/queries/catalog/location-map-query/location-map-query-executor.ts index 2aa60c9b72d..163038fd514 100644 --- a/src/queries/catalog/location-map-query/location-map-query-executor.ts +++ b/src/queries/catalog/location-map-query/location-map-query-executor.ts @@ -42,7 +42,7 @@ export async function executeLocationMapQuery({ input }: BasicQueryData, queries count++; } - const ast = await input.normalizedAst(); + const ast = await input.normalize(); for(const [id, node] of ast.idMap.entries()) { if(node.location && (criteriaOfInterest.size === 0 || criteriaOfInterest.has(id))) { const file = fuzzyFindFile(node, ast.idMap); diff --git a/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts b/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts index f866fa1539e..c37dd010ed9 100644 --- a/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts +++ b/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts @@ -12,6 +12,6 @@ export async function executeNormalizedAstQuery({ input }: BasicQueryData, queri /* there is no sense in measuring a get */ timing: 0 }, - normalized: await input.normalizedAst() + normalized: await input.normalize() }; } diff --git a/src/queries/catalog/origin-query/origin-query-executor.ts b/src/queries/catalog/origin-query/origin-query-executor.ts index c6ebd4c2f11..809899e31f3 100644 --- a/src/queries/catalog/origin-query/origin-query-executor.ts +++ b/src/queries/catalog/origin-query/origin-query-executor.ts @@ -19,7 +19,7 @@ export async function executeResolveValueQuery({ input }: BasicQueryData, querie log.warn(`Duplicate Key for origin-query: ${key}, skipping...`); } - const astId = slicingCriterionToId(key, (await input.normalizedAst()).idMap); + const astId = slicingCriterionToId(key, (await input.normalize()).idMap); if(astId === undefined) { log.warn(`Could not resolve id for ${key}, skipping...`); continue; diff --git a/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts b/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts index 88a3b14abf6..e927da55d58 100644 --- a/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts +++ b/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts @@ -13,7 +13,7 @@ export async function executeResolveValueQuery({ input }: BasicQueryData, querie const results: ResolveValueQueryResult['results'] = {}; const graph = (await input.dataflow()).graph; - const ast = await input.normalizedAst(); + const ast = await input.normalize(); for(const query of queries) { const key = fingerPrintOfQuery(query); diff --git a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts index e795bd2772e..e1a7842aecb 100644 --- a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts +++ b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts @@ -21,13 +21,13 @@ export async function executeStaticSliceQuery({ input }: BasicQueryData, queries } const { criteria, noReconstruction, noMagicComments } = query; const sliceStart = Date.now(); - const slice = staticSlice(await input.dataflow(), await input.normalizedAst(), criteria, query.direction ?? SliceDirection.Backward, input.flowrConfig.solver.slicer?.threshold); + const slice = staticSlice(await input.dataflow(), await input.normalize(), criteria, query.direction ?? SliceDirection.Backward, input.flowrConfig.solver.slicer?.threshold); const sliceEnd = Date.now(); if(noReconstruction) { results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart, cached: false } } }; } else { const reconstructStart = Date.now(); - const reconstruct = reconstructToCode(await input.normalizedAst(), slice.result, noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)); + const reconstruct = reconstructToCode(await input.normalize(), slice.result, noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)); const reconstructEnd = Date.now(); results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart, cached: false } }, diff --git a/src/search/search-executor/search-enrichers.ts b/src/search/search-executor/search-enrichers.ts index 085c4cc50d3..8545851a40a 100644 --- a/src/search/search-executor/search-enrichers.ts +++ b/src/search/search-executor/search-enrichers.ts @@ -201,7 +201,7 @@ export const Enrichments = { const content: CfgInformationSearchContent = { ...prev, - cfg: await data.controlFlow(args.simplificationPasses, true), + cfg: await data.controlflow(args.simplificationPasses, true), }; if(args.checkReachable) { content.reachableNodes = cfgFindAllReachable(content.cfg); diff --git a/src/search/search-executor/search-generators.ts b/src/search/search-executor/search-generators.ts index c80356398a4..d107f26fc3e 100644 --- a/src/search/search-executor/search-generators.ts +++ b/src/search/search-executor/search-generators.ts @@ -44,14 +44,14 @@ async function generateAll(data: FlowrAnalysisInput): Promise { - const normalize = await data.normalizedAst(); + const normalize = await data.normalize(); return [...new Map([...normalize.idMap.values()].map(n => [n.info.id, n])) .values()]; } async function generateGet(input: FlowrAnalysisInput, { filter: { line, column, id, name, nameIsRegex } }: { filter: FlowrSearchGetFilter }): Promise> { - const normalize = await input.normalizedAst(); + const normalize = await input.normalize(); let potentials = (id ? [normalize.idMap.get(id)].filter(isNotUndefined) : await getAllNodes(input) @@ -101,16 +101,16 @@ async function generateFromQuery(input: FlowrAnalysisInput, args: { const nodes = new Set>(); const queryDef = SupportedQueries[query as Query['type']] as SupportedQuery; for(const node of queryDef.flattenInvolvedNodes(content as BaseQueryResult, args.from)) { - nodes.add({ node: (await input.normalizedAst()).idMap.get(node) as RNode }); + nodes.add({ node: (await input.normalize()).idMap.get(node) as RNode }); } nodesByQuery.set(query as Query['type'], nodes); } // enrich elements with query data - const normalize = await input.normalizedAst(); + const normalize = await input.normalize(); const dataflow = await input.dataflow(); - const cfg = await input.controlFlow(); + const cfg = await input.controlflow(); const elements = await new FlowrSearchElements([...nodesByQuery] .flatMap(([_, nodes]) => [...nodes])) @@ -122,7 +122,7 @@ async function generateFromQuery(input: FlowrAnalysisInput, args: { } async function generateCriterion(input: FlowrAnalysisInput, args: { criterion: SlicingCriteria }): Promise> { - const idMap = (await input.normalizedAst()).idMap; + const idMap = (await input.normalize()).idMap; return new FlowrSearchElements( args.criterion.map(c => ({ node: idMap.get(slicingCriterionToId(c, idMap)) as RNodeWithParent })) ); diff --git a/src/search/search-executor/search-transformer.ts b/src/search/search-executor/search-transformer.ts index c6b43ac8ab0..85267a53eab 100644 --- a/src/search/search-executor/search-transformer.ts +++ b/src/search/search-executor/search-transformer.ts @@ -159,9 +159,9 @@ async function getWith[], }): Promise[]>> { const data = { - normalize: await input.normalizedAst(), + normalize: await input.normalize(), dataflow: await input.dataflow(), - cfg: await input.controlFlow(), + cfg: await input.controlflow(), config: input.flowrConfig }; diff --git a/test/functionality/_helper/query.ts b/test/functionality/_helper/query.ts index aa54b9c2da4..3e070bf5a0c 100644 --- a/test/functionality/_helper/query.ts +++ b/test/functionality/_helper/query.ts @@ -90,7 +90,7 @@ export function assertQuery< // eslint-disable-next-line @typescript-eslint/no-unsafe-argument const expectedNormalized = normalizeResults(typeof expected === 'function' ? await expected( { - parse: await analyzer.parseOutput() as ParseStepOutput & PipelinePerStepMetaInformation, + parse: await analyzer.parse() as ParseStepOutput & PipelinePerStepMetaInformation, normalize: await analyzer.normalizedAst(), dataflow: await analyzer.dataflow() } diff --git a/test/functionality/cli/server.test.ts b/test/functionality/cli/server.test.ts index 0c4c7763feb..85dd3902272 100644 --- a/test/functionality/cli/server.test.ts +++ b/test/functionality/cli/server.test.ts @@ -88,7 +88,7 @@ describe('flowr', () => { const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('1 + 1')) .setParser(shell) .build(); - const results = sanitizeAnalysisResults(await analyzer.parseOutput(), await analyzer.normalizedAst(), await analyzer.dataflow()); + const results = sanitizeAnalysisResults(await analyzer.parse(), await analyzer.normalizedAst(), await analyzer.dataflow()); // cfg should not be set as we did not request it assert.isUndefined(response.cfg, 'Expected the cfg to be undefined as we did not request it'); @@ -122,7 +122,7 @@ describe('flowr', () => { const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('1 + 1')) .setParser(shell) .build(); - const results = sanitizeAnalysisResults(await analyzer.parseOutput(), await analyzer.normalizedAst(), await analyzer.dataflow()); + const results = sanitizeAnalysisResults(await analyzer.parse(), await analyzer.normalizedAst(), await analyzer.dataflow()); // cfg should not be set as we did not request it assert.isUndefined(response.cfg, 'Expected the cfg to be undefined as we did not request it'); From aea3a79cdc9ac3543e14768ee5c678d194d1a5c2 Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Fri, 26 Sep 2025 01:07:13 +0200 Subject: [PATCH 63/70] feat: flo's opinionated analyzer caching updates --- src/cli/repl/commands/repl-dataflow.ts | 2 +- src/cli/repl/commands/repl-normalize.ts | 2 +- src/cli/repl/server/connection.ts | 8 +- src/dataflow/extractor.ts | 8 +- src/documentation/doc-util/doc-query.ts | 2 +- src/documentation/doc-util/doc-search.ts | 3 +- src/project/cache/flowr-analyzer-cache.ts | 213 +++++++++++++++--- src/project/cache/flowr-cache.ts | 2 +- src/project/cache/flowr-controlflow-cache.ts | 36 --- src/project/cache/flowr-dataflow-cache.ts | 21 -- .../cache/flowr-normalized-ast-cache.ts | 22 -- src/project/cache/flowr-parse-cache.ts | 26 --- src/project/flowr-analyzer-builder.ts | 49 ++-- src/project/flowr-analyzer.ts | 142 +++--------- .../static-slice-query-executor.ts | 6 +- src/search/flowr-search-executor.ts | 2 +- src/util/collections/objectmap.ts | 8 +- .../controlflow/assert-control-flow-graph.ts | 14 +- test/functionality/_helper/linter.ts | 2 +- test/functionality/_helper/query.ts | 9 +- test/functionality/_helper/search.ts | 5 +- test/functionality/_helper/shell.ts | 7 +- test/functionality/cli/server.test.ts | 4 +- .../control-flow/control-flow-graph.test.ts | 44 ++-- .../slicing/backward/slicing.bench.ts | 7 +- test/functionality/slicing/slicing.bench.ts | 7 +- 26 files changed, 322 insertions(+), 329 deletions(-) delete mode 100644 src/project/cache/flowr-controlflow-cache.ts delete mode 100644 src/project/cache/flowr-dataflow-cache.ts delete mode 100644 src/project/cache/flowr-normalized-ast-cache.ts delete mode 100644 src/project/cache/flowr-parse-cache.ts diff --git a/src/cli/repl/commands/repl-dataflow.ts b/src/cli/repl/commands/repl-dataflow.ts index 3b8f023e8cd..2466bf4cfa5 100644 --- a/src/cli/repl/commands/repl-dataflow.ts +++ b/src/cli/repl/commands/repl-dataflow.ts @@ -6,7 +6,7 @@ import type { PipelinePerStepMetaInformation } from '../../../core/steps/pipelin import { handleString } from '../core'; function formatInfo(out: ReplOutput, type: string, meta: PipelinePerStepMetaInformation ): string { - return out.formatter.format(`Copied ${type} to clipboard (dataflow: ${meta['.meta'].cached ? 'cached' : meta['.meta'].timing + 'ms'}).`, + return out.formatter.format(`Copied ${type} to clipboard (dataflow: ${meta['.meta'].timing + 'ms'}).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); } diff --git a/src/cli/repl/commands/repl-normalize.ts b/src/cli/repl/commands/repl-normalize.ts index 6649b43fe9e..67796e035de 100644 --- a/src/cli/repl/commands/repl-normalize.ts +++ b/src/cli/repl/commands/repl-normalize.ts @@ -6,7 +6,7 @@ import type { PipelinePerStepMetaInformation } from '../../../core/steps/pipelin import { handleString } from '../core'; function formatInfo(out: ReplOutput, type: string, meta: PipelinePerStepMetaInformation): string { - return out.formatter.format(`Copied ${type} to clipboard (normalize: ${meta['.meta'].cached ? 'cached' : meta['.meta'].timing + 'ms'}).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); + return out.formatter.format(`Copied ${type} to clipboard (normalize: ${meta['.meta'].timing + 'ms'}).`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); } export const normalizeCommand: ReplCodeCommand = { diff --git a/src/cli/repl/server/connection.ts b/src/cli/repl/server/connection.ts index bc93530394e..89af02e8ae1 100644 --- a/src/cli/repl/server/connection.ts +++ b/src/cli/repl/server/connection.ts @@ -168,11 +168,11 @@ export class FlowRServerConnection { private async sendFileAnalysisResponse(analyzer: FlowrAnalyzer, message: FileAnalysisRequestMessage): Promise { let cfg: ControlFlowInformation | undefined = undefined; if(message.cfg) { - cfg = await analyzer.controlFlow(); + cfg = await analyzer.controlflow(); } const config = (): QuadSerializationConfiguration => ({ context: message.filename ?? 'unknown', getId: defaultQuadIdGenerator() }); - const sanitizedResults = sanitizeAnalysisResults(await analyzer.parse(), await analyzer.normalizedAst(), await analyzer.dataflow()); + const sanitizedResults = sanitizeAnalysisResults(await analyzer.parse(), await analyzer.normalize(), await analyzer.dataflow()); if(message.format === 'n-quads') { sendMessage(this.socket, { type: 'response-file-analysis', @@ -181,7 +181,7 @@ export class FlowRServerConnection { cfg: cfg ? cfg2quads(cfg, config()) : undefined, results: { parse: await printStepResult(PARSE_WITH_R_SHELL_STEP, await analyzer.parse() as ParseStepOutput, StepOutputFormat.RdfQuads, config()), - normalize: await printStepResult(NORMALIZE, await analyzer.normalizedAst(), StepOutputFormat.RdfQuads, config()), + normalize: await printStepResult(NORMALIZE, await analyzer.normalize(), StepOutputFormat.RdfQuads, config()), dataflow: await printStepResult(STATIC_DATAFLOW, await analyzer.dataflow(), StepOutputFormat.RdfQuads, config()) } }); @@ -338,7 +338,7 @@ export class FlowRServerConnection { } const analyzer = fileInformation.analyzer; - const lineageIds = getLineage(request.criterion, (await analyzer.dataflow()).graph, (await analyzer.normalizedAst()).idMap); + const lineageIds = getLineage(request.criterion, (await analyzer.dataflow()).graph, (await analyzer.normalize()).idMap); sendMessage(this.socket, { type: 'response-lineage', id: request.id, diff --git a/src/dataflow/extractor.ts b/src/dataflow/extractor.ts index c3b06d6b600..67ccd30e0b2 100644 --- a/src/dataflow/extractor.ts +++ b/src/dataflow/extractor.ts @@ -92,10 +92,10 @@ function resolveLinkToSideEffects(ast: NormalizedAst, graph: DataflowGraph) { * For the actual, canonical fold entry point, see {@link processDataflowFor}. */ export function produceDataFlowGraph( - parser: Parser, - request: RParseRequests, - completeAst: NormalizedAst, - config: FlowrConfigOptions, + parser: Parser, + request: RParseRequests, + completeAst: NormalizedAst, + config: FlowrConfigOptions ): DataflowInformation { let firstRequest: RParseRequest; diff --git a/src/documentation/doc-util/doc-query.ts b/src/documentation/doc-util/doc-query.ts index 0cef65f4088..4f116f34d61 100644 --- a/src/documentation/doc-util/doc-query.ts +++ b/src/documentation/doc-util/doc-query.ts @@ -50,7 +50,7 @@ ${collapseResult ? '
Show Results Show Detailed Results as Json diff --git a/src/documentation/doc-util/doc-search.ts b/src/documentation/doc-util/doc-search.ts index 52db701bf67..95142d5a9b0 100644 --- a/src/documentation/doc-util/doc-search.ts +++ b/src/documentation/doc-util/doc-search.ts @@ -7,7 +7,6 @@ import { printDfGraphForCode } from './doc-dfg'; import { codeBlock } from './doc-code'; import { printAsMs } from '../../util/text/time'; import type { FlowrSearchLike } from '../../search/flowr-search-builder'; -import { runSearch } from '../../search/flowr-search-executor'; import { flowrSearchToCode, flowrSearchToMermaid } from '../../search/flowr-search-printer'; import { recoverContent } from '../../r-bridge/lang-4.x/ast/model/processing/node-id'; import { formatRange } from '../../util/mermaid/dfg'; @@ -23,7 +22,7 @@ export async function showSearch(shell: RShell, code: string, search: FlowrSearc const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) .setParser(shell) .build(); - const result = await runSearch(search, analyzer); + const result = await analyzer.runSearch(search); const duration = performance.now() - now; const metaInfo = ` diff --git a/src/project/cache/flowr-analyzer-cache.ts b/src/project/cache/flowr-analyzer-cache.ts index 6cf2b11a841..5cdab092f3a 100644 --- a/src/project/cache/flowr-analyzer-cache.ts +++ b/src/project/cache/flowr-analyzer-cache.ts @@ -1,48 +1,203 @@ import type { KnownParser } from '../../r-bridge/parser'; -import { FlowrParseCache } from './flowr-parse-cache'; -import { FlowrNormalizeCache } from './flowr-normalized-ast-cache'; -import { FlowrDataflowCache } from './flowr-dataflow-cache'; -import type { CacheInvalidationEvent, CacheInvalidationEventReceiver } from './flowr-cache'; -import { CacheInvalidationEventType } from './flowr-cache'; -import { FlowrControlflowCache } from './flowr-controlflow-cache'; -import type { DEFAULT_DATAFLOW_PIPELINE, TREE_SITTER_DATAFLOW_PIPELINE } from '../../core/steps/pipeline/default-pipelines'; - -export interface FlowrAnalyzerCaches { - parse: FlowrParseCache; - normalize: FlowrNormalizeCache; - dataflow: FlowrDataflowCache; - controlflow: FlowrControlflowCache; +import type { CacheInvalidationEvent } from './flowr-cache'; +import { FlowrCache , CacheInvalidationEventType } from './flowr-cache'; +import type { + DEFAULT_DATAFLOW_PIPELINE, + TREE_SITTER_DATAFLOW_PIPELINE +} from '../../core/steps/pipeline/default-pipelines'; +import { + createDataflowPipeline +} from '../../core/steps/pipeline/default-pipelines'; +import type { PipelineExecutor } from '../../core/pipeline-executor'; +import type { FlowrConfigOptions } from '../../config'; +import { defaultConfigOptions } from '../../config'; +import type { RParseRequests } from '../../r-bridge/retriever'; +import type { IdGenerator } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { NoInfo } from '../../r-bridge/lang-4.x/ast/model/model'; +import type { TreeSitterExecutor } from '../../r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; +import type { PipelineOutput } from '../../core/steps/pipeline/pipeline'; +import { assertUnreachable, guard } from '../../util/assert'; +import { ObjectMap } from '../../util/collections/objectmap'; +import type { CfgSimplificationPassName } from '../../control-flow/cfg-simplification'; +import type { ControlFlowInformation } from '../../control-flow/control-flow-graph'; +import { extractCfg, extractCfgQuick } from '../../control-flow/extract-cfg'; + +interface FlowrAnalyzerCacheOptions { + parser: Parser; + config?: FlowrConfigOptions; + request: RParseRequests; + getId?: IdGenerator + overwriteFilePath?: string; } +type AnalyzerPipeline = Parser extends TreeSitterExecutor ? + typeof TREE_SITTER_DATAFLOW_PIPELINE : typeof DEFAULT_DATAFLOW_PIPELINE; +type AnalyzerPipelineExecutor = PipelineExecutor>; + +/* for whatever reason moving the ternary in with `AnalyzerPipeline` just breaks the type system */ +type AnalyzerCacheType = Parser extends TreeSitterExecutor ? Partial> + : Partial>; + +interface ControlFlowCache { + simplified: ObjectMap<[passes: readonly CfgSimplificationPassName[], df: boolean], ControlFlowInformation>, +} /** * This provides the full analyzer caching layer */ -export class FlowrAnalyzerCache implements CacheInvalidationEventReceiver, FlowrAnalyzerCaches { - public readonly parse: FlowrParseCache; - public readonly normalize: FlowrNormalizeCache; - public readonly dataflow: FlowrDataflowCache; - public readonly controlflow: FlowrControlflowCache; +export class FlowrAnalyzerCache extends FlowrCache> { + private args: FlowrAnalyzerCacheOptions; + private pipeline: AnalyzerPipelineExecutor = undefined as unknown as AnalyzerPipelineExecutor; + private controlFlowCache: ControlFlowCache = undefined as unknown as ControlFlowCache; - /** the currently running pipeline */ - private pipeline: typeof DEFAULT_DATAFLOW_PIPELINE | typeof TREE_SITTER_DATAFLOW_PIPELINE; + protected constructor(args: FlowrAnalyzerCacheOptions) { + super(); + this.args = args; + this.initCacheProviders(); + } + + private initCacheProviders() { + this.pipeline = createDataflowPipeline(this.args.parser, { + request: this.args.request, + getId: this.args.getId, + overwriteFilePath: this.args.overwriteFilePath + }, this.args.config ?? defaultConfigOptions) as AnalyzerPipelineExecutor; + this.controlFlowCache = { + simplified: new ObjectMap<[readonly CfgSimplificationPassName[], boolean], ControlFlowInformation>(), + }; + } - protected constructor(parser: Parser) { - this.parse = FlowrParseCache.create(parser); - this.normalize = FlowrNormalizeCache.create(this.parse); - this.dataflow = FlowrDataflowCache.create(this.normalize); - this.controlflow = FlowrControlflowCache.create(this.dataflow); + public static create(data: FlowrAnalyzerCacheOptions): FlowrAnalyzerCache { + return new FlowrAnalyzerCache(data); } - public static create(parser: Parser): FlowrAnalyzerCache { - return new FlowrAnalyzerCache(parser); + public override receive(event: CacheInvalidationEvent): void { + super.receive(event); + switch(event.type) { + case CacheInvalidationEventType.Full: + this.initCacheProviders(); + break; + default: + assertUnreachable(event.type); + } } - public receive(event: CacheInvalidationEvent) { - this.parse.receive(event); + private get(): AnalyzerCacheType { + /* this will do a ref assignment, so indirect force */ + return this.computeIfAbsent(false, () => this.pipeline.getResults(true)); } public reset() { this.receive({ type: CacheInvalidationEventType.Full }); } + + private async runTapeUntil(force: boolean | undefined, until: () => T | undefined): Promise { + if(force) { + this.reset(); + } + let g: T | undefined; + while((g = until()) === undefined && this.pipeline.hasNextStep()) { + await this.pipeline.nextStep(); + } + guard(g !== undefined, 'Could not reach the desired pipeline step, invalid cache state(?)'); + return g; + } + + /** + * Get the parse output for the request, parsing if necessary. + * @param force - Do not use the cache, instead force a new parse. + * + * @see {@link FlowrAnalyzerCache#peekParse} - to get the parse output if already available without triggering a new parse. + */ + public async parse(force?: boolean): Promise['parse']>> { + return this.runTapeUntil(force, () => this.get().parse); + } + + /** + * Get the parse output for the request if already available, otherwise return `undefined`. + * This will not trigger a new parse. + * + * @see {@link FlowrAnalyzerCache#parse} - to get the parse output, parsing if necessary. + */ + public peekParse(): NonNullable['parse']> | undefined { + return this.get().parse; + } + + /** + * Get the normalized abstract syntax tree for the request, normalizing if necessary. + * @param force - Do not use the cache, instead force new analyses. + * @see {@link FlowrAnalyzerCache#peekNormalize} - to get the normalized AST if already available without triggering a new normalization. + */ + public async normalize(force?: boolean): Promise['normalize']>> { + return this.runTapeUntil(force, () => this.get().normalize); + } + + /** + * Get the normalized abstract syntax tree for the request if already available, otherwise return `undefined`. + * This will not trigger a new normalization. + * + * @see {@link FlowrAnalyzerCache#normalize} - to get the normalized AST, normalizing if necessary. + */ + public peekNormalize(): NonNullable['normalize']> | undefined { + return this.get().normalize; + } + + /** + * Get the dataflow graph for the request, computing if necessary. + * @param force - Do not use the cache, instead force new analyses. + * + * @see {@link FlowrAnalyzerCache#peekDataflow} - to get the dataflow graph if already available without triggering a new computation. + */ + public async dataflow(force?: boolean): Promise['dataflow']>> { + return this.runTapeUntil(force, () => this.get().dataflow); + } + + /** + * Get the dataflow graph for the request if already available, otherwise return `undefined`. + * This will not trigger a new computation. + * + * @see {@link FlowrAnalyzerCache#dataflow} - to get the dataflow graph, computing if necessary. + */ + public peekDataflow(): NonNullable['dataflow']> | undefined { + return this.get().dataflow; + } + + /** + * Get the control flow graph (CFG) for the request, computing if necessary. + * @param force - Do not use the cache, instead force new analyses. + * @param useDataflow - Whether to use the dataflow graph for the creation of the CFG. + * @param simplifications - Simplification passes to be applied to the CFG. + */ + public async controlflow(force: boolean | undefined, useDataflow: boolean, simplifications: readonly CfgSimplificationPassName[] | undefined): Promise { + simplifications ??= []; + if(!force) { + const value = this.controlFlowCache.simplified.get([simplifications, useDataflow]); + if(value !== undefined) { + return value; + } + } + + const normalized = await this.normalize(force); + let dataflow: NonNullable>['dataflow'] | undefined = undefined; + if(useDataflow) { + /* if force is active, it will have triggered with normalize */ + dataflow = await this.dataflow(); + } + + const result = simplifications.length === 0 && !useDataflow ? extractCfgQuick(normalized) : + extractCfg(normalized, this.args.config ?? defaultConfigOptions, dataflow?.graph, simplifications); + this.controlFlowCache.simplified.set([simplifications, useDataflow], result); + return result; + } + + /** + * Get the control flow graph (CFG) for the request if already available, otherwise return `undefined`. + * @param useDataflow - Whether to use the dataflow graph for the creation of the CFG. + * @param simplifications - Simplification passes to be applied to the CFG. + * + * @see {@link FlowrAnalyzerCache#controlflow} - to get the control flow graph, computing if necessary. + */ + public peekControlflow(useDataflow: boolean, simplifications: readonly CfgSimplificationPassName[] | undefined): ControlFlowInformation | undefined { + return this.controlFlowCache.simplified.get([simplifications ?? [], useDataflow]); + } } \ No newline at end of file diff --git a/src/project/cache/flowr-cache.ts b/src/project/cache/flowr-cache.ts index eacb06c4f0d..90559973808 100644 --- a/src/project/cache/flowr-cache.ts +++ b/src/project/cache/flowr-cache.ts @@ -50,7 +50,7 @@ export abstract class FlowrCache implements CacheInvalidationEventReceive * Get the cached value or compute it if not present. * This will, by default, not trigger any {@link notifyDependents} calls, as this is only a cache retrieval. */ - public computeIfAbsent(force: boolean | undefined, compute: () => Cache): Cache { + protected computeIfAbsent(force: boolean | undefined, compute: () => Cache): Cache { if(this.value === undefined || force) { this.value = compute(); } diff --git a/src/project/cache/flowr-controlflow-cache.ts b/src/project/cache/flowr-controlflow-cache.ts deleted file mode 100644 index 0c02863842e..00000000000 --- a/src/project/cache/flowr-controlflow-cache.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { FlowrCache } from './flowr-cache'; -import type { KnownParser } from '../../r-bridge/parser'; -import type { ObjectMap } from '../../util/collections/objectmap'; -import type { CfgSimplificationPassName } from '../../control-flow/cfg-simplification'; -import type { ControlFlowInformation } from '../../control-flow/control-flow-graph'; -import type { FlowrDataflowCache } from './flowr-dataflow-cache'; - - -interface ControlFlowCache { - simplified: ObjectMap, - quick: ControlFlowInformation -} - -/** - * Controlflow specific cache. - */ -export class FlowrControlflowCache extends FlowrCache< - ControlFlowCache -> { - - protected constructor(dataflow: FlowrDataflowCache) { - super(); - dataflow.registerDependent(this); - } - - public static create(dataflow: FlowrDataflowCache): FlowrControlflowCache { - return new FlowrControlflowCache(dataflow); - } - // TODO: updates - /* - this.controlFlowInfos = { - simplified: new ObjectMap(), - quick: undefined as unknown as ControlFlowInformation - }; - */ -} \ No newline at end of file diff --git a/src/project/cache/flowr-dataflow-cache.ts b/src/project/cache/flowr-dataflow-cache.ts deleted file mode 100644 index 62aec446a3c..00000000000 --- a/src/project/cache/flowr-dataflow-cache.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { FlowrCache } from './flowr-cache'; -import type { KnownParser } from '../../r-bridge/parser'; -import type { DataflowInformation } from '../../dataflow/info'; -import type { FlowrNormalizeCache } from './flowr-normalized-ast-cache'; - -/** - * Dataflow specific cache. - */ -export class FlowrDataflowCache extends FlowrCache< - DataflowInformation -> { - - protected constructor(normalizer: FlowrNormalizeCache) { - super(); - normalizer.registerDependent(this); - } - - public static create(normalizer: FlowrNormalizeCache): FlowrDataflowCache { - return new FlowrDataflowCache(normalizer); - } -} \ No newline at end of file diff --git a/src/project/cache/flowr-normalized-ast-cache.ts b/src/project/cache/flowr-normalized-ast-cache.ts deleted file mode 100644 index 8eeabbec89b..00000000000 --- a/src/project/cache/flowr-normalized-ast-cache.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { FlowrCache } from './flowr-cache'; -import type { NormalizedAst } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'; -import type { FlowrParseCache } from './flowr-parse-cache'; -import type { KnownParser } from '../../r-bridge/parser'; - -/** - * Flowr normalized AST specific cache. - */ -export class FlowrNormalizeCache extends FlowrCache< - NormalizedAst -> { - - protected constructor(parser: FlowrParseCache) { - super(); - parser.registerDependent(this); - } - - public static create(parser: FlowrParseCache): FlowrNormalizeCache { - /* in the future, with e.g. incrementality, we want to specialize based on the parser */ - return new FlowrNormalizeCache(parser); - } -} \ No newline at end of file diff --git a/src/project/cache/flowr-parse-cache.ts b/src/project/cache/flowr-parse-cache.ts deleted file mode 100644 index 6b3bd3a065a..00000000000 --- a/src/project/cache/flowr-parse-cache.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { FlowrCache } from './flowr-cache'; -import type { KnownParser, ParseStepOutput } from '../../r-bridge/parser'; -import type { PipelinePerStepMetaInformation } from '../../core/steps/pipeline/pipeline'; - -/** - * Flowr Parser specific cache. - */ -export class FlowrParseCache extends FlowrCache< - ParseStepOutput>> & PipelinePerStepMetaInformation -> { - private readonly parser: Parser; - - protected constructor(parser: Parser) { - super(); - this.parser = parser; - } - - public static create(parser: Parser): FlowrParseCache { - /* in the future, with e.g. incrementality, we want to specialize based on the parser */ - return new FlowrParseCache(parser); - } - - public getParser(): Parser { - return this.parser; - } -} \ No newline at end of file diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 2903d94808c..893b0fa6788 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -7,16 +7,41 @@ import { retrieveEngineInstances } from '../engines'; import type { KnownParser } from '../r-bridge/parser'; import type { FlowrAnalyzerPlugin } from './plugins/flowr-analyzer-plugin'; import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize'; +import { guard } from '../util/assert'; /** * Builder for the {@link FlowrAnalyzer}. */ export class FlowrAnalyzerBuilder { - private flowrConfig: DeepWritable = cloneConfig(defaultConfigOptions); - private parser?: KnownParser; - private readonly request: RParseRequests; - private input?: Omit; - private plugins: FlowrAnalyzerPlugin[]; + private flowrConfig: DeepWritable = cloneConfig(defaultConfigOptions); + private parser?: KnownParser; + private request: RParseRequests | undefined; + private input?: Omit; + private plugins: FlowrAnalyzerPlugin[] = []; + + + /** + * Create a new builder instance. + * @param request - The code to analyze + */ + constructor(request?: RParseRequests) { + this.request = request; + } + + /** + * Add one or multiple requests to analyze the builder. + */ + public addRequest(request: RParseRequests) { + if(this.request) { + if(Array.isArray(this.request)) { + this.request = this.request.concat(request); + } else { + this.request = [this.request].concat(request) as RParseRequests; + } + } else { + this.request = request; + } + } /** * Apply an amendment to the configuration the builder currently holds. @@ -52,7 +77,7 @@ export class FlowrAnalyzerBuilder { * @param engine - The engine to use. */ public setEngine(engine: EngineConfig['type']) { - this.flowrConfig.defaultEngine = engine; + (this.flowrConfig.defaultEngine as string) = engine; return this; } @@ -65,16 +90,6 @@ export class FlowrAnalyzerBuilder { return this; } - /** - * Create a new builder instance. - * @param request - The code to analyze. - * @param plugins - The plugins to register. - */ - constructor(request: RParseRequests, plugins?: FlowrAnalyzerPlugin[]) { - this.request = request; - this.plugins = plugins ?? []; - } - /** * Register one or multiple additional plugins. * @param plugin - One or multiple plugins. @@ -105,6 +120,8 @@ export class FlowrAnalyzerBuilder { parser = this.parser ?? engines.engines[engines.default] as KnownParser; } + guard(this.request !== undefined, 'Currently we require at least one request to build an analyzer, please provide one using the constructor or the addRequest method'); + return new FlowrAnalyzer( this.flowrConfig, parser, diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index b48d4e033d5..ba72bb8e0a2 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -2,22 +2,20 @@ import type { FlowrConfigOptions } from '../config'; import type { RParseRequests } from '../r-bridge/retriever'; import type { DEFAULT_DATAFLOW_PIPELINE } from '../core/steps/pipeline/default-pipelines'; -import { - createDataflowPipeline, - createNormalizePipeline, - createParsePipeline -} from '../core/steps/pipeline/default-pipelines'; + + import type { KnownParser, KnownParserName, ParseStepOutput } from '../r-bridge/parser'; import type { Queries, QueryResults, SupportedQueryTypes } from '../queries/query'; import { executeQueries } from '../queries/query'; -import { extractCfg, extractCfgQuick } from '../control-flow/extract-cfg'; import type { ControlFlowInformation } from '../control-flow/control-flow-graph'; import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { DataflowInformation } from '../dataflow/info'; import type { CfgSimplificationPassName } from '../control-flow/cfg-simplification'; import type { PipelineInput, PipelinePerStepMetaInformation } from '../core/steps/pipeline/pipeline'; -import type { NormalizeRequiredInput } from '../core/steps/all/core/10-normalize'; import { FlowrAnalyzerCache } from './cache/flowr-analyzer-cache'; +import type { FlowrSearchLike, SearchOutput } from '../search/flowr-search-builder'; +import type { GetSearchElements } from '../search/flowr-search-executor'; +import { runSearch } from '../search/flowr-search-executor'; /** * Exposes the central analyses and information provided by the {@link FlowrAnalyzer} to the linter, search, and query APIs. @@ -33,19 +31,15 @@ export type FlowrAnalysisInput = { } - /** * Central class for creating analyses in FlowR. * Use the {@link FlowrAnalyzerBuilder} to create a new instance. */ export class FlowrAnalyzer { /** This is the config used for the analyzer */ - public readonly flowrConfig: FlowrConfigOptions; - /** The currently active request that governs this analyzer */ - private readonly request: RParseRequests; + public readonly flowrConfig: FlowrConfigOptions; /** The parser and engine backend */ - private readonly parser: Parser; - private readonly requiredInput: Omit, 'parser' | 'request'>; + private readonly parser: Parser; private readonly cache: FlowrAnalyzerCache; @@ -57,12 +51,10 @@ export class FlowrAnalyzer { * @param request - The code to analyze. * @param requiredInput - Additional parameters used for the analyses. */ - constructor(config: FlowrConfigOptions, parser: Parser, request: RParseRequests, requiredInput: Omit) { + constructor(config: FlowrConfigOptions, parser: Parser, request: RParseRequests, requiredInput: Omit, 'parser' | 'request'>) { this.flowrConfig = config; - this.request = request; this.parser = parser; - this.requiredInput = requiredInput; - this.cache = FlowrAnalyzerCache.create(parser); + this.cache = FlowrAnalyzerCache.create({ parser, config, request, ...requiredInput }); } /** @@ -85,74 +77,24 @@ export class FlowrAnalyzer { * The parse result type depends on the {@link KnownParser} used by the analyzer. * @param force - Do not use the cache, instead force a new parse. */ - public async parse(force?: boolean): Promise>> & PipelinePerStepMetaInformation> { - - return this.cache.parse.computeIfAbsent(force, - () => { - - } - ); - - if(this.parseCache && !force) { - return { - ...this.parseCache, - '.meta': { - cached: true - } - }; - } - - const result = await createParsePipeline( - this.parser, - { request: this.request }, - this.flowrConfig).allRemainingSteps(); - this.parseCache = result.parse as unknown as ParseStepOutput>> & PipelinePerStepMetaInformation; - return this.parseCache; + public async parse(force?: boolean): ReturnType { + return this.cache.parse(force); } /** * Get the normalized abstract syntax tree for the request. * @param force - Do not use the cache, instead force new analyses. */ - public async normalizedAst(force?: boolean): Promise { - if(this.normalizeCache && !force) { - return { - ...this.normalizeCache, - '.meta': { - cached: true - } - }; - } - - const result = await createNormalizePipeline( - this.parser, - { request: this.request, ...this.requiredInput }, - this.flowrConfig).allRemainingSteps(); - this.normalizeCache = result.normalize; - return result.normalize; + public async normalize(force?: boolean): ReturnType { + return this.cache.normalize(force); } /** * Get the dataflow graph for the request. * @param force - Do not use the cache, instead force new analyses. */ - public async dataflow(force?: boolean): Promise { - if(this.dataflowCache && !force) { - return { - ...this.dataflowCache, - '.meta': { - cached: true - } - }; - } - - const result = await createDataflowPipeline( - this.parser, - { request: this.request }, - this.flowrConfig).allRemainingSteps(); - this.dataflowCache = result.dataflow; - this.normalizeCache = result.normalize; - return result.dataflow; + public async dataflow(force?: boolean): ReturnType { + return this.cache.dataflow(force); } /** @@ -161,51 +103,16 @@ export class FlowrAnalyzer { * @param useDataflow - Whether to use the dataflow graph for the creation of the CFG. * @param force - Do not use the cache, instead force new analyses. */ - public async controlFlow(simplifications?: readonly CfgSimplificationPassName[], useDataflow?: boolean, force?: boolean): Promise { - if(!force) { - const value = this.controlFlowInfos.simplified.get(simplifications ?? []); - if(value !== undefined) { - return value; - } - } - - if(force || !this.normalizeCache) { - await this.normalizedAst(force); - } - - if(useDataflow && (force || !this.dataflowCache)) { - await this.dataflow(force); - } - - const result = extractCfg(this.normalizeCache, this.flowrConfig, this.dataflowCache?.graph, simplifications); - this.controlFlowInfos.simplified.set(simplifications ?? [], result); - return result; + public async controlflow(simplifications?: readonly CfgSimplificationPassName[], useDataflow?: boolean, force?: boolean): Promise { + return this.cache.controlflow(force, useDataflow ?? false, simplifications); } /** - * Get a more performant version of the control flow graph. + * Get a quick and dirty control flow graph (CFG) for the request. * @param force - Do not use the cache, instead force new analyses. */ - public async controlFlowQuick(force?: boolean): Promise { - if(!force) { - if(this.controlFlowInfos.quick) { - return this.controlFlowInfos.quick; - } - - // Use the unsimplified CFG if it is already available - const value = this.controlFlowInfos.simplified.get([]); - if(value !== undefined) { - return value; - } - } - - if(force || !this.normalizeCache) { - await this.normalizedAst(force); - } - - const result = extractCfgQuick(this.normalizeCache); - this.controlFlowInfos.quick = result; - return result; + public async controlflowQuick(force?: boolean): Promise { + return this.controlflow(undefined, false, force); } /** @@ -217,4 +124,13 @@ export class FlowrAnalyzer { >(query: Queries): Promise> { return executeQueries({ input: this }, query); } + + /** + * Run a search on the current analysis. + */ + public async runSearch< + Search extends FlowrSearchLike + >(search: Search): Promise>> { + return runSearch(search, this); + } } \ No newline at end of file diff --git a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts index e1a7842aecb..9c310c0d185 100644 --- a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts +++ b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts @@ -24,14 +24,14 @@ export async function executeStaticSliceQuery({ input }: BasicQueryData, queries const slice = staticSlice(await input.dataflow(), await input.normalize(), criteria, query.direction ?? SliceDirection.Backward, input.flowrConfig.solver.slicer?.threshold); const sliceEnd = Date.now(); if(noReconstruction) { - results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart, cached: false } } }; + results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart } } }; } else { const reconstructStart = Date.now(); const reconstruct = reconstructToCode(await input.normalize(), slice.result, noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)); const reconstructEnd = Date.now(); results[key] = { - slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart, cached: false } }, - reconstruct: { ...reconstruct, '.meta': { timing: reconstructEnd - reconstructStart, cached: false } } + slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart } }, + reconstruct: { ...reconstruct, '.meta': { timing: reconstructEnd - reconstructStart } } }; } } diff --git a/src/search/flowr-search-executor.ts b/src/search/flowr-search-executor.ts index f44a8125004..414ad071f27 100644 --- a/src/search/flowr-search-executor.ts +++ b/src/search/flowr-search-executor.ts @@ -6,7 +6,7 @@ import { getTransformer } from './search-executor/search-transformer'; import type { ParentInformation } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { FlowrAnalysisInput } from '../project/flowr-analyzer'; -type GetSearchElements = S extends FlowrSearch ? Elements : never; +export type GetSearchElements = S extends FlowrSearch ? Elements : never; /** * Run a search with the given search query and data. diff --git a/src/util/collections/objectmap.ts b/src/util/collections/objectmap.ts index ed02626636c..5f507babf9a 100644 --- a/src/util/collections/objectmap.ts +++ b/src/util/collections/objectmap.ts @@ -3,24 +3,24 @@ * {@link JSON.stringify} is used to create the actual key for the underlying map. * This can be helpful if value equality is desired. */ -export class ObjectMap { +export class ObjectMap { private readonly internal = new Map(); - private makeKey(key: readonly K[]): string { + private makeKey(key: K): string { return JSON.stringify(key); } /** * Sets a value for a given key. */ - public set(key: readonly K[], v: V): void { + public set(key: K, v: V): void { this.internal.set(this.makeKey(key), v); } /** * Return the value for the key. */ - public get(key: readonly K[]): V | undefined { + public get(key: K): V | undefined { return this.internal.get(this.makeKey(key)); } } diff --git a/test/functionality/_helper/controlflow/assert-control-flow-graph.ts b/test/functionality/_helper/controlflow/assert-control-flow-graph.ts index 3c84d0d633e..a922c2ffd98 100644 --- a/test/functionality/_helper/controlflow/assert-control-flow-graph.ts +++ b/test/functionality/_helper/controlflow/assert-control-flow-graph.ts @@ -40,12 +40,14 @@ export function assertCfg(parser: KnownParser, code: string, partialExpected: Pa .setConfig(config) .setParser(parser) .build(); - let cfg = await analyzer.controlFlow(undefined, true); + let cfg: ControlFlowInformation; if(options?.withBasicBlocks) { - cfg = await analyzer.controlFlow(['to-basic-blocks', 'remove-dead-code', ...options.simplificationPasses ?? []]); + cfg = await analyzer.controlflow(['to-basic-blocks', 'remove-dead-code', ...options.simplificationPasses ?? []], true); } else if(options?.simplificationPasses) { - cfg = await analyzer.controlFlow(options.simplificationPasses ?? []); + cfg = await analyzer.controlflow(options.simplificationPasses ?? [], true); + } else { + cfg = await analyzer.controlflow(undefined, true); } let diff: GraphDifferenceReport | undefined; @@ -64,14 +66,14 @@ export function assertCfg(parser: KnownParser, code: string, partialExpected: Pa }); assert.isTrue(diff.isEqual(), 'graphs differ:' + (diff?.comments() ?? []).join('\n')); if(options?.additionalAsserts) { - options.additionalAsserts(cfg, await analyzer.normalizedAst(), await analyzer.dataflow()); + options.additionalAsserts(cfg, await analyzer.normalize(), await analyzer.dataflow()); } } /* v8 ignore next 7 */ catch(e: unknown) { if(diff) { console.error(diff.comments()); } - console.error(`expected: ${cfgToMermaidUrl(expected, await analyzer.normalizedAst())}`); - console.error(`actual: ${cfgToMermaidUrl(cfg, await analyzer.normalizedAst())}`); + console.error(`expected: ${cfgToMermaidUrl(expected, await analyzer.normalize())}`); + console.error(`actual: ${cfgToMermaidUrl(cfg, await analyzer.normalize())}`); throw e; } }); diff --git a/test/functionality/_helper/linter.ts b/test/functionality/_helper/linter.ts index 3821333b272..6afac8c6772 100644 --- a/test/functionality/_helper/linter.ts +++ b/test/functionality/_helper/linter.ts @@ -62,7 +62,7 @@ export function assertLinter( } if(typeof expected === 'function') { - expected = expected(await analyzer.dataflow(), await analyzer.normalizedAst()); + expected = expected(await analyzer.dataflow(), await analyzer.normalize()); } try { diff --git a/test/functionality/_helper/query.ts b/test/functionality/_helper/query.ts index 3e070bf5a0c..22217ea82e7 100644 --- a/test/functionality/_helper/query.ts +++ b/test/functionality/_helper/query.ts @@ -71,16 +71,21 @@ export function assertQuery< } } + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) .setParser(parser) .build(); + // we run the dfa analysis to make sure normalization post-patches are ready! + await analyzer.dataflow(); + const dummyProject = await getDummyFlowrProject(); const result = await executeQueries({ input: analyzer, libraries: dummyProject.libraries }, queries); + log.info(`total query time: ${result['.meta'].timing.toFixed(0)}ms (~1ms accuracy)`); const normalized = normalizeResults(result); @@ -91,14 +96,14 @@ export function assertQuery< const expectedNormalized = normalizeResults(typeof expected === 'function' ? await expected( { parse: await analyzer.parse() as ParseStepOutput & PipelinePerStepMetaInformation, - normalize: await analyzer.normalizedAst(), + normalize: await analyzer.normalize(), dataflow: await analyzer.dataflow() } ) : expected); assert.deepStrictEqual(normalized, expectedNormalized, 'The result of the query does not match the expected result'); } /* v8 ignore next 3 */ catch(e: unknown) { console.error('Dataflow-Graph', dataflowGraphToMermaidUrl(await analyzer.dataflow())); - console.error('Control-Flow-Graph', cfgToMermaidUrl(extractCfg(await analyzer.normalizedAst(), defaultConfigOptions, (await analyzer.dataflow()).graph), await analyzer.normalizedAst())); + console.error('Control-Flow-Graph', cfgToMermaidUrl(extractCfg(await analyzer.normalize(), defaultConfigOptions, (await analyzer.dataflow()).graph), await analyzer.normalize())); throw e; } }); diff --git a/test/functionality/_helper/search.ts b/test/functionality/_helper/search.ts index f247185e11f..1c0078d557c 100644 --- a/test/functionality/_helper/search.ts +++ b/test/functionality/_helper/search.ts @@ -8,7 +8,6 @@ import { dataflowGraphToMermaidUrl } from '../../../src/core/print/dataflow-prin import type { FlowrSearchLike } from '../../../src/search/flowr-search-builder'; import { getFlowrSearch } from '../../../src/search/flowr-search-builder'; import type { NodeId } from '../../../src/r-bridge/lang-4.x/ast/model/processing/node-id'; -import { runSearch } from '../../../src/search/flowr-search-executor'; import { arrayEqual } from '../../../src/util/collections/arrays'; import { type SingleSlicingCriterion, slicingCriterionToId } from '../../../src/slicing/criterion/parse'; import { guard, isNotUndefined } from '../../../src/util/assert'; @@ -46,7 +45,7 @@ export function assertSearch( .setParser(parser) .build(); dataflow = await analyzer.dataflow(); - ast = await analyzer.normalizedAst(); + ast = await analyzer.normalize(); }); @@ -57,7 +56,7 @@ export function assertSearch( guard(isNotUndefined(ast), 'Normalized AST must be defined'); search = getFlowrSearch(search, optimize); - const result = (await runSearch(search, analyzer)).getElements(); + const result = (await analyzer.runSearch(search)).getElements(); try { if(Array.isArray(expected)) { expected = expected.map(id => { diff --git a/test/functionality/_helper/shell.ts b/test/functionality/_helper/shell.ts index c4d2fa98a36..fbb4bead8d8 100644 --- a/test/functionality/_helper/shell.ts +++ b/test/functionality/_helper/shell.ts @@ -42,7 +42,6 @@ import semver from 'semver/preload'; import { TreeSitterExecutor } from '../../../src/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; import type { PipelineOutput } from '../../../src/core/steps/pipeline/pipeline'; import type { FlowrSearchLike } from '../../../src/search/flowr-search-builder'; -import { runSearch } from '../../../src/search/flowr-search-executor'; import type { ContainerIndex } from '../../../src/dataflow/graph/vertex'; import type { REnvironmentInformation } from '../../../src/dataflow/environments/environment'; import { resolveByName } from '../../../src/dataflow/environments/resolve-by-name'; @@ -380,7 +379,7 @@ export function assertDataflow( expected = await expected(analyzer); } - const normalize = await analyzer.normalizedAst(); + const normalize = await analyzer.normalize(); const dataflow = await analyzer.dataflow(); // assign the same id map to the expected graph, so that resolves work as expected @@ -652,8 +651,8 @@ export function assertContainerIndicesDefinition( .setParser(shell) .build(); const dataflow = await analyzer.dataflow(); - const normalize = await analyzer.normalizedAst(); - const result = (await runSearch(search, analyzer)).getElements(); + const normalize = await analyzer.normalize(); + const result = (await analyzer.runSearch(search)).getElements(); let findIndices: (id: NodeId) => ContainerIndex[] | undefined; if(userConfig.searchIn === 'dfg') { findIndices = id => findInDfg(id, dataflow.graph); diff --git a/test/functionality/cli/server.test.ts b/test/functionality/cli/server.test.ts index 85dd3902272..d4dfd6d3917 100644 --- a/test/functionality/cli/server.test.ts +++ b/test/functionality/cli/server.test.ts @@ -88,7 +88,7 @@ describe('flowr', () => { const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('1 + 1')) .setParser(shell) .build(); - const results = sanitizeAnalysisResults(await analyzer.parse(), await analyzer.normalizedAst(), await analyzer.dataflow()); + const results = sanitizeAnalysisResults(await analyzer.parse(), await analyzer.normalize(), await analyzer.dataflow()); // cfg should not be set as we did not request it assert.isUndefined(response.cfg, 'Expected the cfg to be undefined as we did not request it'); @@ -122,7 +122,7 @@ describe('flowr', () => { const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('1 + 1')) .setParser(shell) .build(); - const results = sanitizeAnalysisResults(await analyzer.parse(), await analyzer.normalizedAst(), await analyzer.dataflow()); + const results = sanitizeAnalysisResults(await analyzer.parse(), await analyzer.normalize(), await analyzer.dataflow()); // cfg should not be set as we did not request it assert.isUndefined(response.cfg, 'Expected the cfg to be undefined as we did not request it'); diff --git a/test/functionality/control-flow/control-flow-graph.test.ts b/test/functionality/control-flow/control-flow-graph.test.ts index 94517f18118..6558ee9cbd5 100644 --- a/test/functionality/control-flow/control-flow-graph.test.ts +++ b/test/functionality/control-flow/control-flow-graph.test.ts @@ -295,8 +295,8 @@ describe('Control Flow Graph', withTreeSitter(parser => { }, { withBasicBlocks: true }); assertCfg(parser, 'if(TRUE) {} else {}', { - entryPoints: [ 'bb-0' ], - exitPoints: [ 'bb-8-exit' ], + entryPoints: ['bb-0'], + exitPoints: ['bb-8-exit'], graph: new ControlFlowGraph() .addVertex({ id: 'bb-0', @@ -426,21 +426,29 @@ describe('Control Flow Graph', withTreeSitter(parser => { }, { withBasicBlocks: true }); - assertCfg(parser, 'f <- function(x) x\nf()', { - entryPoints: [ 'bb-5' ], - exitPoints: [ 'bb-9-exit' ], - graph: new ControlFlowGraph() - .addVertex({ - id: 'bb-8-exit', - type: CfgVertexType.Block, - elems: [ - { id: '8-exit', type: CfgVertexType.EndMarker, root: 8 }, - { id: 7, type: CfgVertexType.Expression }, - { id: 8, type: CfgVertexType.Statement, mid: [0], end: ['8-exit'], callTargets: new Set([5]) }, - { id: '6-exit', type: CfgVertexType.EndMarker, root: 6 } - - ] - }) - }, { expectIsSubgraph: true, withBasicBlocks: true }); + describe.only('conditionals', () => { + assertCfg(parser, 'f <- function(x) x\nf()', { + entryPoints: ['bb-5'], + exitPoints: ['bb-9-exit'], + graph: new ControlFlowGraph() + .addVertex({ + id: 'bb-8-exit', + type: CfgVertexType.Block, + elems: [ + { id: '8-exit', type: CfgVertexType.EndMarker, root: 8 }, + { id: 7, type: CfgVertexType.Expression }, + { + id: 8, + type: CfgVertexType.Statement, + mid: [0], + end: ['8-exit'], + callTargets: new Set([5]) + }, + { id: '6-exit', type: CfgVertexType.EndMarker, root: 6 } + + ] + }) + }, { expectIsSubgraph: true, withBasicBlocks: true }); + }); }); })); diff --git a/test/functionality/slicing/backward/slicing.bench.ts b/test/functionality/slicing/backward/slicing.bench.ts index e182cf2786b..e962ecc970a 100644 --- a/test/functionality/slicing/backward/slicing.bench.ts +++ b/test/functionality/slicing/backward/slicing.bench.ts @@ -2,7 +2,6 @@ import { bench, describe } from 'vitest'; import { TreeSitterExecutor } from '../../../../src/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; import { requestFromInput } from '../../../../src/r-bridge/retriever'; import type { NodeId } from '../../../../src/r-bridge/lang-4.x/ast/model/processing/node-id'; -import { runSearch } from '../../../../src/search/flowr-search-executor'; import { Q } from '../../../../src/search/flowr-search-builder'; import { guard } from '../../../../src/util/assert'; import { staticSlice } from '../../../../src/slicing/static/static-slicer'; @@ -39,10 +38,10 @@ x[2] <- x[1] + x[3] const exec = new TreeSitterExecutor(); const analyzer = await new FlowrAnalyzerBuilder(requestFromInput(code)) .setParser(exec).build(); - result = { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalizedAst() }; - ids = (await runSearch(Q.var('print').first(), analyzer)).getElements().map(n => n.node.info.id); + result = { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalize() }; + ids = (await analyzer.runSearch(Q.var('print').first())).getElements().map(n => n.node.info.id); } - guard(result !== undefined && ids !== undefined, () => 'no result'); + guard(ids !== undefined, () => 'no result'); staticSlice(result.dataflow, result.normalize, [`$${ids[0]}`], SliceDirection.Backward, threshold); }); } diff --git a/test/functionality/slicing/slicing.bench.ts b/test/functionality/slicing/slicing.bench.ts index dd06b27c7e1..963331125ae 100644 --- a/test/functionality/slicing/slicing.bench.ts +++ b/test/functionality/slicing/slicing.bench.ts @@ -2,7 +2,6 @@ import { bench, describe } from 'vitest'; import { TreeSitterExecutor } from '../../../src/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; import { requestFromInput } from '../../../src/r-bridge/retriever'; import type { NodeId } from '../../../src/r-bridge/lang-4.x/ast/model/processing/node-id'; -import { runSearch } from '../../../src/search/flowr-search-executor'; import { Q } from '../../../src/search/flowr-search-builder'; import { guard } from '../../../src/util/assert'; import { staticSlice } from '../../../src/slicing/static/static-slicer'; @@ -41,10 +40,10 @@ for(i in 1:5) { const analyzer = await new FlowrAnalyzerBuilder(request) .setParser(exec) .build(); - result = { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalizedAst() }; - ids = (await runSearch(Q.var('print').first(), analyzer)).getElements().map(n => n.node.info.id); + result = { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalize() }; + ids = (await analyzer.runSearch(Q.var('print').first())).getElements().map(n => n.node.info.id); } - guard(result !== undefined && ids !== undefined, () => 'no result'); + guard(ids !== undefined, () => 'no result'); staticSlice(result.dataflow, result.normalize, [`$${ids[0]}`], SliceDirection.Backward, threshold); }); } From 83fa01d4de35b2ccb3645190bccc6e1475d70b7e Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Fri, 26 Sep 2025 01:14:28 +0200 Subject: [PATCH 64/70] refactor: simplification iteration --- src/cli/repl/commands/repl-query.ts | 8 ++++---- src/project/flowr-project.ts | 4 ++-- .../flowr-analyzer-description-file-plugin.ts | 15 +++++++++------ .../file-plugins/flowr-analyzer-file-plugin.ts | 4 ++-- src/project/plugins/flowr-analyzer-plugin.ts | 5 +++-- ...lyzer-loading-order-description-file-plugin.ts | 8 +++----- ...er-package-versions-description-file-plugin.ts | 6 ++---- test/functionality/_helper/query.ts | 2 +- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index 547c8931546..eee7249fbb1 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -24,7 +24,7 @@ function printHelp(output: ReplOutput) { output.stdout(`With this, ${italic(':query @config', output.formatter)} prints the result of the config query.`); } -async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisInput, remainingArgs: string[]): Promise, processed: {dataflow: DataflowInformation, normalize: NormalizedAst} }> { +async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisInput, remainingArgs: string[]): Promise { const query = remainingArgs.shift(); if(!query) { @@ -64,13 +64,13 @@ async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisInput parsedQuery = [{ type: 'call-context', callName: query }]; } - const dummyProject = await getDummyFlowrProject(); + const dummyProject = getDummyFlowrProject(); return { - query: await Promise.resolve(executeQueries({ + query: await executeQueries({ input: analyzer, libraries: dummyProject.libraries }, - parsedQuery)), + parsedQuery), parsedQuery, processed: { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalize() } }; diff --git a/src/project/flowr-project.ts b/src/project/flowr-project.ts index e404e5a3539..e0e4fbae0d3 100644 --- a/src/project/flowr-project.ts +++ b/src/project/flowr-project.ts @@ -18,7 +18,7 @@ export interface FlowrProject { projectRoot: PathLike; } -export async function getDummyFlowrProject(){ +export function getDummyFlowrProject(): FlowrProject{ const exampleFlowrProject: FlowrProject = { analyzer: {} as FlowrAnalyzer, builder: {} as FlowrAnalyzerBuilder, @@ -33,7 +33,7 @@ export async function getDummyFlowrProject(){ const flowrAnalyzerPackageVersionsDescriptionFilePlugin = new FlowrAnalyzerPackageVersionsDescriptionFilePlugin(); flowrAnalyzerPackageVersionsDescriptionFilePlugin.dependencies = [descriptionFilePlugin]; - await flowrAnalyzerPackageVersionsDescriptionFilePlugin + flowrAnalyzerPackageVersionsDescriptionFilePlugin .processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); exampleFlowrProject.libraries = flowrAnalyzerPackageVersionsDescriptionFilePlugin.packages; diff --git a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts index 82bd955eb09..268592c47f1 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts @@ -4,6 +4,9 @@ import type { FlowrAnalysisInput } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import { parseDCF } from '../../../util/files'; +import { log } from '../../../util/log'; + +const analyzerDescriptionLog = log.getSubLogger({ name: 'flowr-analyzer-description-log' }); export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin { public readonly name = 'flowr-analyzer-description-file-plugin'; @@ -12,15 +15,15 @@ export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin public readonly dependencies: FlowrAnalyzerPlugin[] = []; public information: Map = new Map(); - public async processor(_analyzer: FlowrAnalysisInput, _pluginConfig: FlowrConfigOptions): Promise { + public processor(_analyzer: FlowrAnalysisInput, _pluginConfig: FlowrConfigOptions): void { if(this.files.length === 0) { - throw new Error('FlowrAnalyzerDescriptionFilePlugin: No DESCRIPTION file found.'); + analyzerDescriptionLog.error(Error('No DESCRIPTION file found.')); + return; } - if(this.files.length > 1){ - throw new Error('FlowrAnalyzerDescriptionFilePlugin: Found more than one DESCRIPTION file.'); + if(this.files.length > 1) { + analyzerDescriptionLog.error(Error('Found more than one DESCRIPTION file.')); + return; } this.information = parseDCF(this.files[0]); - - return Promise.resolve(); } } \ No newline at end of file diff --git a/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts index 5a5eea24347..e30de6ef4e7 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts @@ -5,7 +5,7 @@ export abstract class FlowrAnalyzerFilePlugin extends FlowrAnalyzerPlugin { public readonly type = 'file'; protected files: PathLike[] = []; - public addFiles(...files: PathLike[]): void { - this.files.push(...files); + public addFiles(...files: readonly PathLike[]): void { + this.files = this.files.concat(files); } } \ No newline at end of file diff --git a/src/project/plugins/flowr-analyzer-plugin.ts b/src/project/plugins/flowr-analyzer-plugin.ts index 34243668c54..04ee6dd0206 100644 --- a/src/project/plugins/flowr-analyzer-plugin.ts +++ b/src/project/plugins/flowr-analyzer-plugin.ts @@ -2,6 +2,7 @@ import type { SemVer } from 'semver'; import type { FlowrAnalysisInput } from '../flowr-analyzer'; import type { FlowrConfigOptions } from '../../config'; import type { PathLike } from 'fs'; +import type { AsyncOrSync } from 'ts-essentials'; export type PluginType = 'package-versions' | 'loading-order' | 'scoping' | 'file'; @@ -12,7 +13,7 @@ export interface FlowrAnalyzerPluginInterface { readonly type: PluginType; dependencies: FlowrAnalyzerPlugin[]; - processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): Promise; + processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): AsyncOrSync; } export abstract class FlowrAnalyzerPlugin implements FlowrAnalyzerPluginInterface { @@ -27,6 +28,6 @@ export abstract class FlowrAnalyzerPlugin implements FlowrAnalyzerPluginInterfac this.rootPath = rootPath; } - public abstract processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): Promise; + public abstract processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): AsyncOrSync; } diff --git a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts index d9749a4f4be..9fd49d5f49b 100644 --- a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts +++ b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts @@ -13,13 +13,11 @@ export class FlowrAnalyzerLoadingOrderDescriptionFilePlugin extends FlowrAnalyze dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; descriptionFile: Map = new Map(); - async processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): Promise { + processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): void { const plugin = this.dependencies[0] as FlowrAnalyzerDescriptionFilePlugin; - await plugin.processor(analyzer, pluginConfig); + plugin.processor(analyzer, pluginConfig); this.descriptionFile = plugin.information; - this.loadingOrder = this.descriptionFile?.get('Collate')?.slice() || []; - - return Promise.resolve(undefined); + this.loadingOrder = this.descriptionFile?.get('Collate')?.slice() ?? []; } } \ No newline at end of file diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts index 79ec77609e3..80cc3f57f7f 100644 --- a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts @@ -15,15 +15,13 @@ export class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnal dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; descriptionFile: Map = new Map(); - async processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): Promise { + processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): void { const plugin = this.dependencies[0] as FlowrAnalyzerDescriptionFilePlugin; - await plugin.processor(analyzer, pluginConfig); + plugin.processor(analyzer, pluginConfig); this.descriptionFile = plugin.information; this.retrieveVersionsFromField('Depends', 'r'); this.retrieveVersionsFromField('Imports', 'package'); - - return Promise.resolve(undefined); } private retrieveVersionsFromField(field: string, type?: PackageType): void{ diff --git a/test/functionality/_helper/query.ts b/test/functionality/_helper/query.ts index 22217ea82e7..10a210a1b0e 100644 --- a/test/functionality/_helper/query.ts +++ b/test/functionality/_helper/query.ts @@ -79,7 +79,7 @@ export function assertQuery< // we run the dfa analysis to make sure normalization post-patches are ready! await analyzer.dataflow(); - const dummyProject = await getDummyFlowrProject(); + const dummyProject = getDummyFlowrProject(); const result = await executeQueries({ input: analyzer, libraries: dummyProject.libraries From 9eb16947499868b3a5ea0ebb4730ace49e11fcbc Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Fri, 26 Sep 2025 01:22:45 +0200 Subject: [PATCH 65/70] refactor: defer early force in query print --- src/cli/repl/commands/repl-cfg.ts | 4 +-- src/cli/repl/commands/repl-main.ts | 4 +-- src/cli/repl/commands/repl-query.ts | 10 +++---- src/dataflow/graph/dataflowgraph-builder.ts | 22 ++++++++-------- src/documentation/doc-util/doc-query.ts | 2 +- src/linter/linter-executor.ts | 4 +-- src/project/flowr-analyzer.ts | 2 +- .../flowr-analyzer-description-file-plugin.ts | 4 +-- src/project/plugins/flowr-analyzer-plugin.ts | 6 ++--- ...r-loading-order-description-file-plugin.ts | 4 +-- ...ackage-versions-description-file-plugin.ts | 4 +-- src/queries/base-query-format.ts | 4 +-- .../call-context-query-format.ts | 7 +++-- .../cluster-query/cluster-query-format.ts | 4 +-- .../config-query/config-query-format.ts | 2 +- .../control-flow-query-format.ts | 4 +-- .../dataflow-lens-query-format.ts | 2 +- .../dataflow-query/dataflow-query-format.ts | 2 +- .../dependencies-query-format.ts | 2 +- .../df-shape-query/df-shape-query-format.ts | 2 +- .../happens-before-query-format.ts | 2 +- .../id-map-query/id-map-query-format.ts | 4 +-- .../lineage-query/lineage-query-format.ts | 2 +- .../linter-query/linter-query-format.ts | 2 +- .../location-map-query-format.ts | 2 +- .../normalized-ast-query-format.ts | 2 +- .../origin-query/origin-query-format.ts | 2 +- .../project-query/project-query-format.ts | 2 +- .../resolve-value-query-format.ts | 2 +- .../search-query/search-query-format.ts | 2 +- .../static-slice-query-format.ts | 2 +- src/queries/query-print.ts | 11 +++++--- src/queries/query.ts | 5 ++-- src/search/flowr-search-executor.ts | 4 +-- src/search/flowr-search.ts | 4 +-- .../search-executor/search-enrichers.ts | 4 +-- .../search-executor/search-generators.ts | 14 +++++----- src/search/search-executor/search-mappers.ts | 8 +++--- .../search-executor/search-transformer.ts | 26 +++++++++---------- test/functionality/_helper/shell.ts | 4 +-- 40 files changed, 99 insertions(+), 100 deletions(-) diff --git a/src/cli/repl/commands/repl-cfg.ts b/src/cli/repl/commands/repl-cfg.ts index 406a7d565dd..384aef8fd7f 100644 --- a/src/cli/repl/commands/repl-cfg.ts +++ b/src/cli/repl/commands/repl-cfg.ts @@ -6,14 +6,14 @@ import type { ControlFlowInformation } from '../../../control-flow/control-flow- import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; import type { CfgSimplificationPassName } from '../../../control-flow/cfg-simplification'; import { DefaultCfgSimplificationOrder } from '../../../control-flow/cfg-simplification'; -import type { FlowrAnalysisInput } from '../../../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../../project/flowr-analyzer'; import { handleString } from '../core'; function formatInfo(out: ReplOutput, type: string): string { return out.formatter.format(`Copied ${type} to clipboard.`, { color: Colors.White, effect: ColorEffect.Foreground, style: FontStyles.Italic }); } -async function produceAndPrintCfg(analyzer: FlowrAnalysisInput, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string) { +async function produceAndPrintCfg(analyzer: FlowrAnalysisProvider, output: ReplOutput, simplifications: readonly CfgSimplificationPassName[], cfgConverter: (cfg: ControlFlowInformation, ast: NormalizedAst) => string) { const cfg = await analyzer.controlflow([...DefaultCfgSimplificationOrder, ...simplifications]); const normalizedAst = await analyzer.normalize(); const mermaid = cfgConverter(cfg, normalizedAst); diff --git a/src/cli/repl/commands/repl-main.ts b/src/cli/repl/commands/repl-main.ts index 29e2c755ed2..321759c2389 100644 --- a/src/cli/repl/commands/repl-main.ts +++ b/src/cli/repl/commands/repl-main.ts @@ -2,7 +2,7 @@ import type { OutputFormatter } from '../../../util/text/ansi'; import { formatter } from '../../../util/text/ansi'; import type { KnownParser } from '../../../r-bridge/parser'; import type { FlowrConfigOptions } from '../../../config'; -import type { FlowrAnalysisInput } from '../../../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../../project/flowr-analyzer'; /** * Defines the main interface for output of the repl. @@ -47,7 +47,7 @@ export interface ReplCommandInformation { */ export interface ReplCodeCommandInformation { output: ReplOutput, - analyzer: FlowrAnalysisInput + analyzer: FlowrAnalysisProvider remainingArgs: string[] } diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index eee7249fbb1..944afb5305a 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -7,10 +7,8 @@ import type { Query, QueryResults, SupportedQuery, SupportedQueryTypes } from '. import { AnyQuerySchema, executeQueries, QueriesSchema, SupportedQueries } from '../../../queries/query'; import { jsonReplacer } from '../../../util/json'; import { asciiSummaryOfQueryResult } from '../../../queries/query-print'; -import type { FlowrAnalysisInput } from '../../../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../../project/flowr-analyzer'; import { getDummyFlowrProject } from '../../../project/flowr-project'; -import type { DataflowInformation } from '../../../dataflow/info'; -import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; function printHelp(output: ReplOutput) { @@ -24,7 +22,7 @@ function printHelp(output: ReplOutput) { output.stdout(`With this, ${italic(':query @config', output.formatter)} prints the result of the config query.`); } -async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisInput, remainingArgs: string[]): Promise { +async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisProvider, remainingArgs: string[]): Promise { const query = remainingArgs.shift(); if(!query) { @@ -72,7 +70,7 @@ async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisInput libraries: dummyProject.libraries }, parsedQuery), parsedQuery, - processed: { dataflow: await analyzer.dataflow(), normalize: await analyzer.normalize() } + analyzer }; } @@ -101,7 +99,7 @@ export const queryCommand: ReplCodeCommand = { const results = await processQueryArgs(output, analyzer, remainingArgs); const totalEnd = Date.now(); if(results) { - output.stdout(asciiSummaryOfQueryResult(ansiFormatter, totalEnd - totalStart, results.query, results.processed, results.parsedQuery)); + output.stdout(await asciiSummaryOfQueryResult(ansiFormatter, totalEnd - totalStart, results.query, results.analyzer, results.parsedQuery)); } } }; diff --git a/src/dataflow/graph/dataflowgraph-builder.ts b/src/dataflow/graph/dataflowgraph-builder.ts index 174982c511a..4e87d17e8c4 100644 --- a/src/dataflow/graph/dataflowgraph-builder.ts +++ b/src/dataflow/graph/dataflowgraph-builder.ts @@ -17,7 +17,7 @@ import { DefaultBuiltinConfig, getDefaultProcessor } from '../environments/defau import type { FlowrSearchLike } from '../../search/flowr-search-builder'; import { runSearch } from '../../search/flowr-search-executor'; import { guard } from '../../util/assert'; -import type { FlowrAnalysisInput } from '../../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../project/flowr-analyzer'; export function emptyGraph(idMap?: AstIdMap) { return new DataflowGraphBuilder(idMap); @@ -204,7 +204,7 @@ export class DataflowGraphBuilder extends DataflowGraph { return this.addEdge(normalizeIdToNumberIfPossible(from), normalizeIdToNumberIfPossible(to as NodeId), type); } - private async queryHelper(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput, type: EdgeType) { + private async queryHelper(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisProvider, type: EdgeType) { let fromId: NodeId; if('nodeId' in from) { fromId = from.nodeId; @@ -243,7 +243,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * @param to - Either a node id or a query to find the node id. * @param input - The input to search in i.e. the dataflow graph. */ - public readsQuery(from: FromQueryParam, to: ToQueryParam, input: FlowrAnalysisInput) { + public readsQuery(from: FromQueryParam, to: ToQueryParam, input: FlowrAnalysisProvider) { return this.queryHelper(from, to, input, EdgeType.Reads); } @@ -262,7 +262,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public definedByQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { + public definedByQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisProvider) { return this.queryHelper(from, to, data, EdgeType.DefinedBy); } @@ -280,7 +280,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public callsQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { + public callsQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisProvider) { return this.queryHelper(from, to, data, EdgeType.Calls); } @@ -298,7 +298,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public returnsQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { + public returnsQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisProvider) { return this.queryHelper(from, to, data, EdgeType.Returns); } @@ -316,7 +316,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public definesOnCallQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { + public definesOnCallQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisProvider) { return this.queryHelper(from, to, data, EdgeType.DefinesOnCall); } @@ -334,7 +334,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public definedByOnCallQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { + public definedByOnCallQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisProvider) { return this.queryHelper(from, to, data, EdgeType.DefinedByOnCall); } @@ -352,7 +352,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public argumentQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { + public argumentQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisProvider) { return this.queryHelper(from, to, data, EdgeType.Argument); } @@ -370,7 +370,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public nseQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { + public nseQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisProvider) { return this.queryHelper(from, to, data, EdgeType.NonStandardEvaluation); } @@ -388,7 +388,7 @@ export class DataflowGraphBuilder extends DataflowGraph { * * @see {@link DataflowGraphBuilder#readsQuery|readsQuery} for parameters. */ - public sideEffectOnCallQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisInput) { + public sideEffectOnCallQuery(from: FromQueryParam, to: ToQueryParam, data: FlowrAnalysisProvider) { return this.queryHelper(from, to, data, EdgeType.SideEffectOnCall); } diff --git a/src/documentation/doc-util/doc-query.ts b/src/documentation/doc-util/doc-query.ts index 4f116f34d61..eaa1e59ddc4 100644 --- a/src/documentation/doc-util/doc-query.ts +++ b/src/documentation/doc-util/doc-query.ts @@ -50,7 +50,7 @@ ${collapseResult ? '
Show Results Show Detailed Results as Json diff --git a/src/linter/linter-executor.ts b/src/linter/linter-executor.ts index 23150fb4721..3b90133853f 100644 --- a/src/linter/linter-executor.ts +++ b/src/linter/linter-executor.ts @@ -4,9 +4,9 @@ import type { LintingResults, LintingRule } from './linter-format'; import { runSearch } from '../search/flowr-search-executor'; import type { DeepPartial } from 'ts-essentials'; import { deepMergeObject } from '../util/objects'; -import type { FlowrAnalysisInput } from '../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../project/flowr-analyzer'; -export async function executeLintingRule(ruleName: Name, input: FlowrAnalysisInput, lintingRuleConfig?: DeepPartial>): Promise> { +export async function executeLintingRule(ruleName: Name, input: FlowrAnalysisProvider, lintingRuleConfig?: DeepPartial>): Promise> { try { const rule = LintingRules[ruleName] as unknown as LintingRule, LintingRuleMetadata, LintingRuleConfig>; const fullConfig = deepMergeObject>(rule.info.defaultConfig, lintingRuleConfig); diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index ba72bb8e0a2..f0d86000fde 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -21,7 +21,7 @@ import { runSearch } from '../search/flowr-search-executor'; * Exposes the central analyses and information provided by the {@link FlowrAnalyzer} to the linter, search, and query APIs. * This allows us to exchange the underlying implementation of the analyzer without affecting the APIs. */ -export type FlowrAnalysisInput = { +export type FlowrAnalysisProvider = { parserName(): string parse(force?: boolean): Promise>> & PipelinePerStepMetaInformation> normalize(force?: boolean): Promise; diff --git a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts index 268592c47f1..612dfcf3595 100644 --- a/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts +++ b/src/project/plugins/file-plugins/flowr-analyzer-description-file-plugin.ts @@ -1,6 +1,6 @@ import { FlowrAnalyzerFilePlugin } from './flowr-analyzer-file-plugin'; import { SemVer } from 'semver'; -import type { FlowrAnalysisInput } from '../../flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import { parseDCF } from '../../../util/files'; @@ -15,7 +15,7 @@ export class FlowrAnalyzerDescriptionFilePlugin extends FlowrAnalyzerFilePlugin public readonly dependencies: FlowrAnalyzerPlugin[] = []; public information: Map = new Map(); - public processor(_analyzer: FlowrAnalysisInput, _pluginConfig: FlowrConfigOptions): void { + public processor(_analyzer: FlowrAnalysisProvider, _pluginConfig: FlowrConfigOptions): void { if(this.files.length === 0) { analyzerDescriptionLog.error(Error('No DESCRIPTION file found.')); return; diff --git a/src/project/plugins/flowr-analyzer-plugin.ts b/src/project/plugins/flowr-analyzer-plugin.ts index 04ee6dd0206..4edd7973710 100644 --- a/src/project/plugins/flowr-analyzer-plugin.ts +++ b/src/project/plugins/flowr-analyzer-plugin.ts @@ -1,5 +1,5 @@ import type { SemVer } from 'semver'; -import type { FlowrAnalysisInput } from '../flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../flowr-analyzer'; import type { FlowrConfigOptions } from '../../config'; import type { PathLike } from 'fs'; import type { AsyncOrSync } from 'ts-essentials'; @@ -13,7 +13,7 @@ export interface FlowrAnalyzerPluginInterface { readonly type: PluginType; dependencies: FlowrAnalyzerPlugin[]; - processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): AsyncOrSync; + processor(analyzer: FlowrAnalysisProvider, pluginConfig: FlowrConfigOptions): AsyncOrSync; } export abstract class FlowrAnalyzerPlugin implements FlowrAnalyzerPluginInterface { @@ -28,6 +28,6 @@ export abstract class FlowrAnalyzerPlugin implements FlowrAnalyzerPluginInterfac this.rootPath = rootPath; } - public abstract processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): AsyncOrSync; + public abstract processor(analyzer: FlowrAnalysisProvider, pluginConfig: FlowrConfigOptions): AsyncOrSync; } diff --git a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts index 9fd49d5f49b..c46ab500b6e 100644 --- a/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts +++ b/src/project/plugins/loading-order-plugins/flowr-analyzer-loading-order-description-file-plugin.ts @@ -1,7 +1,7 @@ import { FlowrAnalyzerDescriptionFilePlugin } from '../file-plugins/flowr-analyzer-description-file-plugin'; import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import { SemVer } from 'semver'; -import type { FlowrAnalysisInput } from '../../flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; import { FlowrAnalyzerLoadingOrderPlugin } from './flowr-analyzer-loading-order-plugin'; @@ -13,7 +13,7 @@ export class FlowrAnalyzerLoadingOrderDescriptionFilePlugin extends FlowrAnalyze dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; descriptionFile: Map = new Map(); - processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): void { + processor(analyzer: FlowrAnalysisProvider, pluginConfig: FlowrConfigOptions): void { const plugin = this.dependencies[0] as FlowrAnalyzerDescriptionFilePlugin; plugin.processor(analyzer, pluginConfig); this.descriptionFile = plugin.information; diff --git a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts index 80cc3f57f7f..5b4fecacc54 100644 --- a/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts +++ b/src/project/plugins/package-version-plugins/flowr-analyzer-package-versions-description-file-plugin.ts @@ -2,7 +2,7 @@ import { FlowrAnalyzerPackageVersionsPlugin } from './flowr-analyzer-package-ver import { FlowrAnalyzerDescriptionFilePlugin } from '../file-plugins/flowr-analyzer-description-file-plugin'; import type { FlowrAnalyzerPlugin } from '../flowr-analyzer-plugin'; import { SemVer } from 'semver'; -import type { FlowrAnalysisInput } from '../../flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../flowr-analyzer'; import type { FlowrConfigOptions } from '../../../config'; import type { PackageType } from './package'; import { Package } from './package'; @@ -15,7 +15,7 @@ export class FlowrAnalyzerPackageVersionsDescriptionFilePlugin extends FlowrAnal dependencies: FlowrAnalyzerPlugin[] = [new FlowrAnalyzerDescriptionFilePlugin()]; descriptionFile: Map = new Map(); - processor(analyzer: FlowrAnalysisInput, pluginConfig: FlowrConfigOptions): void { + processor(analyzer: FlowrAnalysisProvider, pluginConfig: FlowrConfigOptions): void { const plugin = this.dependencies[0] as FlowrAnalyzerDescriptionFilePlugin; plugin.processor(analyzer, pluginConfig); this.descriptionFile = plugin.information; diff --git a/src/queries/base-query-format.ts b/src/queries/base-query-format.ts index aea2d5ecdd6..0f057d85c70 100644 --- a/src/queries/base-query-format.ts +++ b/src/queries/base-query-format.ts @@ -1,5 +1,5 @@ import type { Package } from '../project/plugins/package-version-plugins/package'; -import type { FlowrAnalysisInput } from '../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../project/flowr-analyzer'; import type { SemVer } from 'semver'; export interface BaseQueryFormat { @@ -18,5 +18,5 @@ export interface BaseQueryResult { export interface BasicQueryData { readonly lib?: Record; readonly libraries?: Package[]; - readonly input: FlowrAnalysisInput; + readonly input: FlowrAnalysisProvider; } diff --git a/src/queries/catalog/call-context-query/call-context-query-format.ts b/src/queries/catalog/call-context-query/call-context-query-format.ts index 92791530e3e..0fa287b93df 100644 --- a/src/queries/catalog/call-context-query/call-context-query-format.ts +++ b/src/queries/catalog/call-context-query/call-context-query-format.ts @@ -12,8 +12,7 @@ import type { DataflowGraph } from '../../../dataflow/graph/graph'; import type { DataflowGraphVertexInfo } from '../../../dataflow/graph/vertex'; import type { CascadeAction } from './cascade-action'; import type { NoInfo } from '../../../r-bridge/lang-4.x/ast/model/model'; -import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; -import type { DataflowInformation } from '../../../dataflow/info'; +import type { FlowrAnalysisProvider } from '../../../project/flowr-analyzer'; export interface FileFilter { /** @@ -126,10 +125,10 @@ const CallContextQueryLinkTo = Joi.object({ export const CallContextQueryDefinition = { executor: executeCallContextQueries, - asciiSummarizer: (formatter: OutputFormatter, processed: {dataflow: DataflowInformation, normalize: NormalizedAst}, queryResults: BaseQueryResult, result: string[]) => { + asciiSummarizer: async(formatter: OutputFormatter, analyzer: FlowrAnalysisProvider, queryResults: BaseQueryResult, result: string[]) => { const out = queryResults as CallContextQueryResult; result.push(`Query: ${bold('call-context', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); - result.push(asciiCallContext(formatter, out, processed.normalize.idMap)); + result.push(asciiCallContext(formatter, out, (await analyzer.normalize()).idMap)); return true; }, schema: Joi.object({ diff --git a/src/queries/catalog/cluster-query/cluster-query-format.ts b/src/queries/catalog/cluster-query/cluster-query-format.ts index 6bb938f0f03..ee74c45fd02 100644 --- a/src/queries/catalog/cluster-query/cluster-query-format.ts +++ b/src/queries/catalog/cluster-query/cluster-query-format.ts @@ -22,7 +22,7 @@ export interface DataflowClusterQueryResult extends BaseQueryResult { export const ClusterQueryDefinition = { executor: executeDataflowClusterQuery, - asciiSummarizer: (formatter, processed, queryResults, result) => { + asciiSummarizer: async(formatter, analyzer, queryResults, result) => { const out = queryResults as QueryResults<'dataflow-cluster'>['dataflow-cluster']; result.push(`Query: ${bold('dataflow-cluster', formatter)} (${out['.meta'].timing.toFixed(0)}ms)`); result.push(` â•° Found ${out.clusters.length} cluster${out.clusters.length === 1 ? '' : 's'}`); @@ -31,7 +31,7 @@ export const ClusterQueryDefinition = { let suffix = ''; if(formatter === markdownFormatter) { suffix = `([marked](${ - graphToMermaidUrl(processed.dataflow.graph, false, new Set(cluster.members)) + graphToMermaidUrl((await analyzer.dataflow()).graph, false, new Set(cluster.members)) }))`; } result.push(` â•° ${unknownSideEffects} {${summarizeIdsIfTooLong(formatter, cluster.members)}} ${suffix}`); diff --git a/src/queries/catalog/config-query/config-query-format.ts b/src/queries/catalog/config-query/config-query-format.ts index b332d28339e..b25159ad82f 100644 --- a/src/queries/catalog/config-query/config-query-format.ts +++ b/src/queries/catalog/config-query/config-query-format.ts @@ -85,7 +85,7 @@ function configQueryLineParser(line: readonly string[], _config: FlowrConfigOpti export const ConfigQueryDefinition = { executor: executeConfigQuery, - asciiSummarizer: (formatter: OutputFormatter, _processed: unknown, queryResults: BaseQueryResult, result: string[]) => { + asciiSummarizer: (formatter: OutputFormatter, _analyzer: unknown, queryResults: BaseQueryResult, result: string[]) => { const out = queryResults as ConfigQueryResult; result.push(`Query: ${bold('config', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); result.push(` â•° Config:\n${JSON.stringify(out.config, jsonReplacer, 4)}`); diff --git a/src/queries/catalog/control-flow-query/control-flow-query-format.ts b/src/queries/catalog/control-flow-query/control-flow-query-format.ts index 690e56ede11..e5e3f4d4d83 100644 --- a/src/queries/catalog/control-flow-query/control-flow-query-format.ts +++ b/src/queries/catalog/control-flow-query/control-flow-query-format.ts @@ -30,10 +30,10 @@ export interface ControlFlowQueryResult extends BaseQueryResult { export const ControlFlowQueryDefinition = { executor: executeControlFlowQuery, - asciiSummarizer: (formatter, processed, queryResults, result) => { + asciiSummarizer: async(formatter, analyzer, queryResults, result) => { const out = queryResults as QueryResults<'control-flow'>['control-flow']; result.push(`Query: ${bold('control-flow', formatter)} (${out['.meta'].timing.toFixed(0)}ms)`); - result.push(` â•° CFG: ${cfgToMermaidUrl(out.controlFlow, processed.normalize)}`); + result.push(` â•° CFG: ${cfgToMermaidUrl(out.controlFlow, await analyzer.normalize())}`); return true; }, schema: Joi.object({ diff --git a/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-format.ts b/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-format.ts index 9d2abcc6f15..7722bb61f70 100644 --- a/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-format.ts +++ b/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-format.ts @@ -21,7 +21,7 @@ export interface DataflowLensQueryResult extends BaseQueryResult { export const DataflowLensQueryDefinition = { executor: executeDataflowLensQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'dataflow-lens'>['dataflow-lens']; result.push(`Query: ${bold('dataflow-lens', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); result.push(` â•° [Simplified Graph](${graphToMermaidUrl(out.simplifiedGraph, false, undefined, true)})`); diff --git a/src/queries/catalog/dataflow-query/dataflow-query-format.ts b/src/queries/catalog/dataflow-query/dataflow-query-format.ts index c2a5b853d6c..8b7606d8bc2 100644 --- a/src/queries/catalog/dataflow-query/dataflow-query-format.ts +++ b/src/queries/catalog/dataflow-query/dataflow-query-format.ts @@ -22,7 +22,7 @@ export interface DataflowQueryResult extends BaseQueryResult { export const DataflowQueryDefinition = { executor: executeDataflowQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'dataflow'>['dataflow']; result.push(`Query: ${bold('dataflow', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); result.push(` â•° [Dataflow Graph](${graphToMermaidUrl(out.graph)})`); diff --git a/src/queries/catalog/dependencies-query/dependencies-query-format.ts b/src/queries/catalog/dependencies-query/dependencies-query-format.ts index 09127b5aad3..5c126e1b368 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-format.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-format.ts @@ -133,7 +133,7 @@ const functionInfoSchema: Joi.ArraySchema = Joi.array().items(Joi.object({ export const DependenciesQueryDefinition = { executor: executeDependenciesQuery, - asciiSummarizer: (formatter, _processed, queryResults, result, queries) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result, queries) => { const out = queryResults as DependenciesQueryResult; result.push(`Query: ${bold('dependencies', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); for(const [category, value] of Object.entries(getAllCategories(queries as DependenciesQuery[]))) { diff --git a/src/queries/catalog/df-shape-query/df-shape-query-format.ts b/src/queries/catalog/df-shape-query/df-shape-query-format.ts index 6f7be8ba53d..cf7b57cd104 100644 --- a/src/queries/catalog/df-shape-query/df-shape-query-format.ts +++ b/src/queries/catalog/df-shape-query/df-shape-query-format.ts @@ -22,7 +22,7 @@ export interface DfShapeQueryResult extends BaseQueryResult { export const DfShapeQueryDefinition = { executor: executeDfShapeQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'df-shape'>['df-shape']; result.push(`Query: ${bold('df-shape', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); result.push(...out.domains.entries().take(20).map(([key, domain]) => { diff --git a/src/queries/catalog/happens-before-query/happens-before-query-format.ts b/src/queries/catalog/happens-before-query/happens-before-query-format.ts index bd0cb34a555..5ccf92961bb 100644 --- a/src/queries/catalog/happens-before-query/happens-before-query-format.ts +++ b/src/queries/catalog/happens-before-query/happens-before-query-format.ts @@ -19,7 +19,7 @@ export interface HappensBeforeQueryResult extends BaseQueryResult { export const HappensBeforeQueryDefinition = { executor: executeHappensBefore, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'happens-before'>['happens-before']; result.push(`Query: ${bold('happens-before', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); for(const [key, value] of Object.entries(out.results)) { diff --git a/src/queries/catalog/id-map-query/id-map-query-format.ts b/src/queries/catalog/id-map-query/id-map-query-format.ts index c9741bd60f4..1b7c57a4190 100644 --- a/src/queries/catalog/id-map-query/id-map-query-format.ts +++ b/src/queries/catalog/id-map-query/id-map-query-format.ts @@ -18,10 +18,10 @@ export interface IdMapQueryResult extends BaseQueryResult { export const IdMapQueryDefinition = { executor: executeIdMapQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'id-map'>['id-map']; result.push(`Query: ${bold('id-map', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); - result.push(` â•° Id List: {${summarizeIdsIfTooLong(formatter, [...out.idMap.keys()])}}`); + result.push(` â•° Id List: {${summarizeIdsIfTooLong(formatter, Array.from(out.idMap.keys()))}}`); return true; }, schema: Joi.object({ diff --git a/src/queries/catalog/lineage-query/lineage-query-format.ts b/src/queries/catalog/lineage-query/lineage-query-format.ts index f0b543e7a76..a0a59b0094d 100644 --- a/src/queries/catalog/lineage-query/lineage-query-format.ts +++ b/src/queries/catalog/lineage-query/lineage-query-format.ts @@ -24,7 +24,7 @@ export interface LineageQueryResult extends BaseQueryResult { export const LineageQueryDefinition = { executor: executeLineageQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'lineage'>['lineage']; result.push(`Query: ${bold('lineage', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); for(const [criteria, lineage] of Object.entries(out.lineages)) { diff --git a/src/queries/catalog/linter-query/linter-query-format.ts b/src/queries/catalog/linter-query/linter-query-format.ts index 17423052d61..4fb9e1ff9cc 100644 --- a/src/queries/catalog/linter-query/linter-query-format.ts +++ b/src/queries/catalog/linter-query/linter-query-format.ts @@ -30,7 +30,7 @@ export interface LinterQueryResult extends BaseQueryResult { export const LinterQueryDefinition = { executor: executeLinterQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'linter'>['linter']; result.push(`Query: ${bold('linter', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); for(const [ruleName, results] of Object.entries(out.results)) { diff --git a/src/queries/catalog/location-map-query/location-map-query-format.ts b/src/queries/catalog/location-map-query/location-map-query-format.ts index 5f1417d9e2a..55f8f255319 100644 --- a/src/queries/catalog/location-map-query/location-map-query-format.ts +++ b/src/queries/catalog/location-map-query/location-map-query-format.ts @@ -26,7 +26,7 @@ export interface LocationMapQueryResult extends BaseQueryResult { export const LocationMapQueryDefinition = { executor: executeLocationMapQuery, - asciiSummarizer: (formatter: OutputFormatter, _processed: unknown, queryResults: BaseQueryResult, result: string[]) => { + asciiSummarizer: (formatter: OutputFormatter, _analyzer: unknown, queryResults: BaseQueryResult, result: string[]) => { const out = queryResults as LocationMapQueryResult; result.push(`Query: ${bold('location-map', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); result.push(' â•° File List:'); diff --git a/src/queries/catalog/normalized-ast-query/normalized-ast-query-format.ts b/src/queries/catalog/normalized-ast-query/normalized-ast-query-format.ts index 65ea160e271..10790f7b8df 100644 --- a/src/queries/catalog/normalized-ast-query/normalized-ast-query-format.ts +++ b/src/queries/catalog/normalized-ast-query/normalized-ast-query-format.ts @@ -20,7 +20,7 @@ export interface NormalizedAstQueryResult extends BaseQueryResult { export const NormalizedAstQueryDefinition = { executor: executeNormalizedAstQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'normalized-ast'>['normalized-ast']; result.push(`Query: ${bold('normalized-ast', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); result.push(` â•° [Normalized AST](${normalizedAstToMermaidUrl(out.normalized.ast)})`); diff --git a/src/queries/catalog/origin-query/origin-query-format.ts b/src/queries/catalog/origin-query/origin-query-format.ts index 6238e132d05..7fc421ea18c 100644 --- a/src/queries/catalog/origin-query/origin-query-format.ts +++ b/src/queries/catalog/origin-query/origin-query-format.ts @@ -23,7 +23,7 @@ export interface OriginQueryResult extends BaseQueryResult { export const OriginQueryDefinition = { executor: executeResolveValueQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'origin'>['origin']; result.push(`Query: ${bold('origin', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); for(const [criteria, obj] of Object.entries(out.results)) { diff --git a/src/queries/catalog/project-query/project-query-format.ts b/src/queries/catalog/project-query/project-query-format.ts index 2d618bc29bf..5c01b8dad4b 100644 --- a/src/queries/catalog/project-query/project-query-format.ts +++ b/src/queries/catalog/project-query/project-query-format.ts @@ -16,7 +16,7 @@ export interface ProjectQueryResult extends BaseQueryResult { export const ProjectQueryDefinition = { executor: executeProjectQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'project'>['project']; result.push(`Query: ${bold('project', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); result.push(` â•° Contains ${out.files.length} file${out.files.length === 1 ? '' : 's'}`); diff --git a/src/queries/catalog/resolve-value-query/resolve-value-query-format.ts b/src/queries/catalog/resolve-value-query/resolve-value-query-format.ts index e114260fd83..9220f7eecd3 100644 --- a/src/queries/catalog/resolve-value-query/resolve-value-query-format.ts +++ b/src/queries/catalog/resolve-value-query/resolve-value-query-format.ts @@ -22,7 +22,7 @@ export interface ResolveValueQueryResult extends BaseQueryResult { export const ResolveValueQueryDefinition = { executor: executeResolveValueQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'resolve-value'>['resolve-value']; result.push(`Query: ${bold('resolve-value', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); for(const [fingerprint, obj] of Object.entries(out.results)) { diff --git a/src/queries/catalog/search-query/search-query-format.ts b/src/queries/catalog/search-query/search-query-format.ts index 68c7e34bf33..1f3dcbce449 100644 --- a/src/queries/catalog/search-query/search-query-format.ts +++ b/src/queries/catalog/search-query/search-query-format.ts @@ -21,7 +21,7 @@ export interface SearchQueryResult extends BaseQueryResult { export const SearchQueryDefinition = { executor: executeSearch, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'search'>['search']; result.push(`Query: ${bold('search', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); for(const [, { ids, search }] of out.results.entries()) { diff --git a/src/queries/catalog/static-slice-query/static-slice-query-format.ts b/src/queries/catalog/static-slice-query/static-slice-query-format.ts index 3acc7d0acc0..3f44a7cc6d4 100644 --- a/src/queries/catalog/static-slice-query/static-slice-query-format.ts +++ b/src/queries/catalog/static-slice-query/static-slice-query-format.ts @@ -44,7 +44,7 @@ export interface StaticSliceQueryResult extends BaseQueryResult { export const StaticSliceQueryDefinition = { executor: executeStaticSliceQuery, - asciiSummarizer: (formatter, _processed, queryResults, result) => { + asciiSummarizer: (formatter, _analyzer, queryResults, result) => { const out = queryResults as QueryResults<'static-slice'>['static-slice']; result.push(`Query: ${bold('static-slice', formatter)} (${printAsMs(out['.meta'].timing, 0)})`); for(const [fingerprint, obj] of Object.entries(out.results)) { diff --git a/src/queries/query-print.ts b/src/queries/query-print.ts index 95b67f02e8b..2bbbb03ce12 100644 --- a/src/queries/query-print.ts +++ b/src/queries/query-print.ts @@ -8,8 +8,8 @@ import type { CallContextQuerySubKindResult } from './catalog/call-context-query import type { BaseQueryMeta, BaseQueryResult } from './base-query-format'; import { printAsMs } from '../util/text/time'; import { isBuiltIn } from '../dataflow/environments/built-in'; -import type { AstIdMap, NormalizedAst, ParentInformation } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; -import type { DataflowInformation } from '../dataflow/info'; +import type { AstIdMap, ParentInformation } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; +import type { FlowrAnalysisProvider } from '../project/flowr-analyzer'; function nodeString(nodeId: NodeId | { id: NodeId, info?: object}, formatter: OutputFormatter, idMap: AstIdMap): string { const isObj = typeof nodeId === 'object' && nodeId !== null && 'id' in nodeId; @@ -76,7 +76,10 @@ export function summarizeIdsIfTooLong(formatter: OutputFormatter, ids: readonly return formatter === markdownFormatter ? textWithTooltip(acc, JSON.stringify(ids)) : acc; } -export function asciiSummaryOfQueryResult(formatter: OutputFormatter, totalInMs: number, results: QueryResults, processed: {dataflow: DataflowInformation, normalize: NormalizedAst}, queries: Queries): string { +export async function asciiSummaryOfQueryResult( + formatter: OutputFormatter, totalInMs: number, results: QueryResults, + analyzer: FlowrAnalysisProvider, queries: Queries +): Promise { const result: string[] = []; for(const [query, queryResults] of Object.entries(results)) { @@ -86,7 +89,7 @@ export function asciiSummaryOfQueryResult(formatt const queryType = SupportedQueries[query as SupportedQueryTypes]; const relevantQueries = queries.filter(q => q.type === query as SupportedQueryTypes) as Query[]; - if(queryType.asciiSummarizer(formatter, processed, queryResults as BaseQueryResult, result, relevantQueries)) { + if(await queryType.asciiSummarizer(formatter, analyzer, queryResults as BaseQueryResult, result, relevantQueries)) { continue; } diff --git a/src/queries/query.ts b/src/queries/query.ts index cba9b5ae898..ea6bf1c8255 100644 --- a/src/queries/query.ts +++ b/src/queries/query.ts @@ -42,12 +42,11 @@ import { LinterQueryDefinition } from './catalog/linter-query/linter-query-forma import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id'; import type { ControlFlowQuery } from './catalog/control-flow-query/control-flow-query-format'; import { ControlFlowQueryDefinition } from './catalog/control-flow-query/control-flow-query-format'; -import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; -import type { DataflowInformation } from '../dataflow/info'; import type { DfShapeQuery } from './catalog/df-shape-query/df-shape-query-format'; import { DfShapeQueryDefinition } from './catalog/df-shape-query/df-shape-query-format'; import type { AsyncOrSync, Writable } from 'ts-essentials'; import type { FlowrConfigOptions } from '../config'; +import type { FlowrAnalysisProvider } from '../project/flowr-analyzer'; /** * These are all queries that can be executed from within flowR @@ -88,7 +87,7 @@ export interface SupportedQuery string[] /** optional query construction from an, e.g., repl line */ fromLine?: (splitLine: readonly string[], config: FlowrConfigOptions) => Query | Query[] | undefined - asciiSummarizer: (formatter: OutputFormatter, processed: {dataflow: DataflowInformation, normalize: NormalizedAst}, queryResults: BaseQueryResult, resultStrings: string[], query: readonly Query[]) => boolean + asciiSummarizer: (formatter: OutputFormatter, analyzer: FlowrAnalysisProvider, queryResults: BaseQueryResult, resultStrings: string[], query: readonly Query[]) => AsyncOrSync schema: Joi.ObjectSchema /** * Flattens the involved query nodes to be added to a flowR search when the {@link fromQuery} function is used based on the given result after this query is executed. diff --git a/src/search/flowr-search-executor.ts b/src/search/flowr-search-executor.ts index 414ad071f27..1691d853521 100644 --- a/src/search/flowr-search-executor.ts +++ b/src/search/flowr-search-executor.ts @@ -4,7 +4,7 @@ import type { FlowrSearchElements } from './flowr-search'; import { getGenerator } from './search-executor/search-generators'; import { getTransformer } from './search-executor/search-transformer'; import type { ParentInformation } from '../r-bridge/lang-4.x/ast/model/processing/decorate'; -import type { FlowrAnalysisInput } from '../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../project/flowr-analyzer'; export type GetSearchElements = S extends FlowrSearch ? Elements : never; @@ -13,7 +13,7 @@ export type GetSearchElements = S extends FlowrSearch( search: S, - input: FlowrAnalysisInput + input: FlowrAnalysisProvider ): Promise>> { const s = getFlowrSearch(search); diff --git a/src/search/flowr-search.ts b/src/search/flowr-search.ts index cd138dc9828..affc7df63d0 100644 --- a/src/search/flowr-search.ts +++ b/src/search/flowr-search.ts @@ -10,7 +10,7 @@ import type { EnrichmentSearchContent } from './search-executor/search-enrichers'; import { Enrichments } from './search-executor/search-enrichers'; -import type { FlowrAnalysisInput } from '../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../project/flowr-analyzer'; /** * Yes, for now we do technically not need a wrapper around the RNode, but this allows us to attach caches etc. @@ -101,7 +101,7 @@ export class FlowrSearchElements(data: FlowrAnalysisInput, enrichment: E, args?: EnrichmentSearchArguments): Promise { + public async enrich(data: FlowrAnalysisProvider, enrichment: E, args?: EnrichmentSearchArguments): Promise { const enrichmentData = Enrichments[enrichment] as unknown as EnrichmentData, EnrichmentElementArguments, EnrichmentSearchContent, EnrichmentSearchArguments>; if(enrichmentData.enrichSearch !== undefined) { this.enrichments = { diff --git a/src/search/search-executor/search-enrichers.ts b/src/search/search-executor/search-enrichers.ts index 8545851a40a..468016f3e0d 100644 --- a/src/search/search-executor/search-enrichers.ts +++ b/src/search/search-executor/search-enrichers.ts @@ -22,7 +22,7 @@ import type { Query, QueryResult } from '../../queries/query'; import type { CfgSimplificationPassName } from '../../control-flow/cfg-simplification'; import { cfgFindAllReachable, DefaultCfgSimplificationOrder } from '../../control-flow/cfg-simplification'; import type { AsyncOrSync, AsyncOrSyncType } from 'ts-essentials'; -import type { FlowrAnalysisInput } from '../../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../project/flowr-analyzer'; import type { DataflowInformation } from '../../dataflow/info'; import { promoteCallName } from '../../queries/catalog/call-context-query/call-context-query-executor'; @@ -32,7 +32,7 @@ export interface EnrichmentData, search: FlowrSearchElements, data: {dataflow: DataflowInformation, normalize: NormalizedAst, cfg: ControlFlowInformation}, args: ElementArguments | undefined, previousValue: ElementContent | undefined) => AsyncOrSync - readonly enrichSearch?: (search: FlowrSearchElements, data: FlowrAnalysisInput, args: SearchArguments | undefined, previousValue: SearchContent | undefined) => AsyncOrSync + readonly enrichSearch?: (search: FlowrSearchElements, data: FlowrAnalysisProvider, args: SearchArguments | undefined, previousValue: SearchContent | undefined) => AsyncOrSync /** * The mapping function used by the {@link Mapper.Enrichment} mapper. */ diff --git a/src/search/search-executor/search-generators.ts b/src/search/search-executor/search-generators.ts index d107f26fc3e..4c26a2134b3 100644 --- a/src/search/search-executor/search-generators.ts +++ b/src/search/search-executor/search-generators.ts @@ -10,7 +10,7 @@ import { executeQueries, SupportedQueries } from '../../queries/query'; import type { BaseQueryResult } from '../../queries/base-query-format'; import type { RNode } from '../../r-bridge/lang-4.x/ast/model/model'; import { enrichElement, Enrichment } from './search-enrichers'; -import type { FlowrAnalysisInput } from '../../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../project/flowr-analyzer'; /** * This is a union of all possible generator node types @@ -38,19 +38,19 @@ export const generators = { 'from-query': generateFromQuery } as const; -async function generateAll(data: FlowrAnalysisInput): Promise> { +async function generateAll(data: FlowrAnalysisProvider): Promise> { return new FlowrSearchElements((await getAllNodes(data)) .map(node => ({ node }))); } -async function getAllNodes(data: FlowrAnalysisInput): Promise { +async function getAllNodes(data: FlowrAnalysisProvider): Promise { const normalize = await data.normalize(); return [...new Map([...normalize.idMap.values()].map(n => [n.info.id, n])) .values()]; } -async function generateGet(input: FlowrAnalysisInput, { filter: { line, column, id, name, nameIsRegex } }: { filter: FlowrSearchGetFilter }): Promise> { +async function generateGet(input: FlowrAnalysisProvider, { filter: { line, column, id, name, nameIsRegex } }: { filter: FlowrSearchGetFilter }): Promise> { const normalize = await input.normalize(); let potentials = (id ? [normalize.idMap.get(id)].filter(isNotUndefined) : @@ -83,11 +83,11 @@ async function generateGet(input: FlowrAnalysisInput, { filter: { line, column, return new FlowrSearchElements(potentials.map(node => ({ node }))); } -function generateFrom(_input: FlowrAnalysisInput, args: { from: FlowrSearchElement | FlowrSearchElement[] }): FlowrSearchElements { +function generateFrom(_input: FlowrAnalysisProvider, args: { from: FlowrSearchElement | FlowrSearchElement[] }): FlowrSearchElements { return new FlowrSearchElements(Array.isArray(args.from) ? args.from : [args.from]); } -async function generateFromQuery(input: FlowrAnalysisInput, args: { +async function generateFromQuery(input: FlowrAnalysisProvider, args: { from: readonly Query[] }): Promise[]>> { const result = await executeQueries({ input }, args.from); @@ -121,7 +121,7 @@ async function generateFromQuery(input: FlowrAnalysisInput, args: { }))) as unknown as FlowrSearchElements[]>; } -async function generateCriterion(input: FlowrAnalysisInput, args: { criterion: SlicingCriteria }): Promise> { +async function generateCriterion(input: FlowrAnalysisProvider, args: { criterion: SlicingCriteria }): Promise> { const idMap = (await input.normalize()).idMap; return new FlowrSearchElements( args.criterion.map(c => ({ node: idMap.get(slicingCriterionToId(c, idMap)) as RNodeWithParent })) diff --git a/src/search/search-executor/search-mappers.ts b/src/search/search-executor/search-mappers.ts index 4ae87a8f7a6..1a5638312c7 100644 --- a/src/search/search-executor/search-mappers.ts +++ b/src/search/search-executor/search-mappers.ts @@ -4,20 +4,20 @@ import type { Enrichment, EnrichmentData, EnrichmentElementContent } from './sea import { enrichmentContent, Enrichments } from './search-enrichers'; import type { MergeableRecord } from '../../util/objects'; -import type { FlowrAnalysisInput } from '../../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../project/flowr-analyzer'; export enum Mapper { Enrichment = 'enrichment' } export interface MapperData { - mapper: (e: FlowrSearchElement, data: FlowrAnalysisInput, args: Arguments) => FlowrSearchElement[] + mapper: (e: FlowrSearchElement, data: FlowrAnalysisProvider, args: Arguments) => FlowrSearchElement[] } export type MapperArguments = typeof Mappers[M] extends MapperData ? Arguments : never; const Mappers = { [Mapper.Enrichment]: { - mapper: (e: FlowrSearchElement, _data: FlowrAnalysisInput, enrichment: Enrichment) => { + mapper: (e: FlowrSearchElement, _data: FlowrAnalysisProvider, enrichment: Enrichment) => { const enrichmentData = Enrichments[enrichment] as unknown as EnrichmentData>; const content = enrichmentContent(e, enrichment); return content !== undefined ? enrichmentData.mapper?.(content) ?? [] : []; @@ -26,6 +26,6 @@ const Mappers = { } as const; export function map, MapperType extends Mapper>( - e: Element, data: FlowrAnalysisInput, mapper: MapperType, args: MapperArguments): Element[] { + e: Element, data: FlowrAnalysisProvider, mapper: MapperType, args: MapperArguments): Element[] { return (Mappers[mapper] as MapperData>).mapper(e, data, args) as Element[]; } diff --git a/src/search/search-executor/search-transformer.ts b/src/search/search-executor/search-transformer.ts index 85267a53eab..c3386084397 100644 --- a/src/search/search-executor/search-transformer.ts +++ b/src/search/search-executor/search-transformer.ts @@ -12,7 +12,7 @@ import { enrichElement } from './search-enrichers'; import type { Mapper, MapperArguments } from './search-mappers'; import { map } from './search-mappers'; import type { ElementOf } from 'ts-essentials'; -import type { FlowrAnalysisInput } from '../../project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../project/flowr-analyzer'; /** @@ -101,23 +101,23 @@ type CascadeEmpty[], NewE Elements extends [] ? FlowrSearchElements : FlowrSearchElements; function getFirst[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE + data: FlowrAnalysisProvider, elements: FSE ): CascadeEmpty { return elements.mutate(e => [getFirstByLocation(e)] as Elements) as unknown as CascadeEmpty; } function getLast[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE): CascadeEmpty]> { + data: FlowrAnalysisProvider, elements: FSE): CascadeEmpty]> { return elements.mutate(e => [getLastByLocation(e)] as Elements) as unknown as CascadeEmpty]>; } function getIndex[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE, { index }: { index: number }): CascadeEmpty { + data: FlowrAnalysisProvider, elements: FSE, { index }: { index: number }): CascadeEmpty { return elements.mutate(e => [sortFully(e)[index]] as Elements) as unknown as CascadeEmpty; } function getSelect[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE, { select }: { select: number[] }): CascadeEmpty { + data: FlowrAnalysisProvider, elements: FSE, { select }: { select: number[] }): CascadeEmpty { return elements.mutate(e => { sortFully(e); return select.map(i => e[i]).filter(isNotUndefined) as Elements; @@ -125,7 +125,7 @@ function getSelect[], FSE } function getTail[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE): CascadeEmpty> { + data: FlowrAnalysisProvider, elements: FSE): CascadeEmpty> { return elements.mutate(e => { const first = getFirstByLocation(e); return e.filter(el => el !== first) as Elements; @@ -133,17 +133,17 @@ function getTail[], FSE e } function getTake[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE, { count }: { count: number }): CascadeEmpty> { + data: FlowrAnalysisProvider, elements: FSE, { count }: { count: number }): CascadeEmpty> { return elements.mutate(e => sortFully(e).slice(0, count) as Elements) as unknown as CascadeEmpty>; } function getSkip[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE, { count }: { count: number }): CascadeEmpty> { + data: FlowrAnalysisProvider, elements: FSE, { count }: { count: number }): CascadeEmpty> { return elements.mutate(e => sortFully(e).slice(count) as Elements) as unknown as CascadeEmpty>; } async function getFilter[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE, { filter }: { + data: FlowrAnalysisProvider, elements: FSE, { filter }: { filter: FlowrFilterExpression }): Promise> { const dataflow = await data.dataflow(); @@ -153,7 +153,7 @@ async function getFilter[ } async function getWith[], FSE extends FlowrSearchElements>( - input: FlowrAnalysisInput, elements: FSE, { info, args }: { + input: FlowrAnalysisProvider, elements: FSE, { info, args }: { info: Enrichment, args?: EnrichmentElementArguments }): Promise[]>> { @@ -171,7 +171,7 @@ async function getWith[], } function getMap[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE, { mapper, args }: { mapper: Mapper, args: MapperArguments }): FlowrSearchElements { + data: FlowrAnalysisProvider, elements: FSE, { mapper, args }: { mapper: Mapper, args: MapperArguments }): FlowrSearchElements { return elements.mutate( elements => elements.flatMap(e => map(e, data, mapper, args)) as Elements ) as unknown as FlowrSearchElements; @@ -179,7 +179,7 @@ function getMap[], FSE ex async function getMerge[], FSE extends FlowrSearchElements>( /* search has to be unknown because it is a recursive type */ - data: FlowrAnalysisInput, elements: FSE, other: { + data: FlowrAnalysisProvider, elements: FSE, other: { search: unknown[], generator: FlowrSearchGeneratorNode }): Promise[]>> { @@ -188,7 +188,7 @@ async function getMerge[] } function getUnique[], FSE extends FlowrSearchElements>( - data: FlowrAnalysisInput, elements: FSE): CascadeEmpty { + data: FlowrAnalysisProvider, elements: FSE): CascadeEmpty { return elements.mutate(e => e.reduce((acc, cur) => { if(!acc.some(el => el.node.id === cur.node.id)) { diff --git a/test/functionality/_helper/shell.ts b/test/functionality/_helper/shell.ts index fbb4bead8d8..bcf0e5aa833 100644 --- a/test/functionality/_helper/shell.ts +++ b/test/functionality/_helper/shell.ts @@ -53,7 +53,7 @@ import { assertCfgSatisfiesProperties } from '../../../src/control-flow/cfg-prop import type { FlowrConfigOptions } from '../../../src/config'; import { cloneConfig, defaultConfigOptions } from '../../../src/config'; import { FlowrAnalyzerBuilder } from '../../../src/project/flowr-analyzer-builder'; -import type { FlowrAnalysisInput } from '../../../src/project/flowr-analyzer'; +import type { FlowrAnalysisProvider } from '../../../src/project/flowr-analyzer'; import type { KnownParser } from '../../../src/r-bridge/parser'; import { SliceDirection } from '../../../src/core/steps/all/static-slicing/00-slice'; @@ -360,7 +360,7 @@ export function assertDataflow( name: string | TestLabel, shell: RShell, input: string | RParseRequests, - expected: DataflowGraph | ((input: FlowrAnalysisInput) => Promise), + expected: DataflowGraph | ((input: FlowrAnalysisProvider) => Promise), userConfig?: Partial, startIndexForDeterministicIds = 0, config = cloneConfig(defaultConfigOptions) From 034b1341e64ccac542718245408bed5a577972a7 Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Fri, 26 Sep 2025 01:26:25 +0200 Subject: [PATCH 66/70] refactor: query improvements for projects --- src/cli/repl/commands/repl-query.ts | 2 +- src/project/flowr-analyzer.ts | 2 +- src/queries/base-query-format.ts | 2 +- .../call-context-query-executor.ts | 17 +++++++++-------- .../cluster-query/cluster-query-executor.ts | 4 ++-- .../config-query/config-query-executor.ts | 6 +++--- .../control-flow-query-executor.ts | 4 ++-- .../dataflow-lens-query-executor.ts | 4 ++-- .../dataflow-query/dataflow-query-executor.ts | 4 ++-- .../dependencies-query-executor.ts | 10 +++++----- .../dependencies-query-format.ts | 2 +- .../df-shape-query/df-shape-query-executor.ts | 10 +++++----- .../happens-before-query-executor.ts | 4 ++-- .../id-map-query/id-map-query-executor.ts | 4 ++-- .../lineage-query/lineage-query-executor.ts | 4 ++-- .../linter-query/linter-query-executor.ts | 4 ++-- .../location-map-query-executor.ts | 6 +++--- .../normalized-ast-query-executor.ts | 4 ++-- .../origin-query/origin-query-executor.ts | 6 +++--- .../project-query/project-query-executor.ts | 4 ++-- .../resolve-value-query-executor.ts | 8 ++++---- .../search-query/search-query-executor.ts | 4 ++-- .../static-slice-query-executor.ts | 6 +++--- src/search/search-executor/search-generators.ts | 2 +- test/functionality/_helper/query.ts | 2 +- 25 files changed, 63 insertions(+), 62 deletions(-) diff --git a/src/cli/repl/commands/repl-query.ts b/src/cli/repl/commands/repl-query.ts index 944afb5305a..9daea17a85c 100644 --- a/src/cli/repl/commands/repl-query.ts +++ b/src/cli/repl/commands/repl-query.ts @@ -66,7 +66,7 @@ async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisProvi return { query: await executeQueries({ - input: analyzer, + analyzer: analyzer, libraries: dummyProject.libraries }, parsedQuery), parsedQuery, diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index f0d86000fde..ca4aa2902b6 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -122,7 +122,7 @@ export class FlowrAnalyzer { public async query< Types extends SupportedQueryTypes = SupportedQueryTypes >(query: Queries): Promise> { - return executeQueries({ input: this }, query); + return executeQueries({ analyzer: this }, query); } /** diff --git a/src/queries/base-query-format.ts b/src/queries/base-query-format.ts index 0f057d85c70..806f075f127 100644 --- a/src/queries/base-query-format.ts +++ b/src/queries/base-query-format.ts @@ -18,5 +18,5 @@ export interface BaseQueryResult { export interface BasicQueryData { readonly lib?: Record; readonly libraries?: Package[]; - readonly input: FlowrAnalysisProvider; + readonly analyzer: FlowrAnalysisProvider; } diff --git a/src/queries/catalog/call-context-query/call-context-query-executor.ts b/src/queries/catalog/call-context-query/call-context-query-executor.ts index 877429e8ab0..dd199eb4d38 100644 --- a/src/queries/catalog/call-context-query/call-context-query-executor.ts +++ b/src/queries/catalog/call-context-query/call-context-query-executor.ts @@ -25,7 +25,7 @@ function isQuoted(node: NodeId, graph: DataflowGraph): boolean { if(vertex === undefined) { return false; } - return [...vertex.values()].some(({ types }) => edgeIncludesType(types, EdgeType.NonStandardEvaluation)); + return vertex.values().some(({ types }) => edgeIncludesType(types, EdgeType.NonStandardEvaluation)); } function makeReport(collector: TwoLayerCollector): CallContextQueryKindResult { @@ -140,9 +140,10 @@ function retrieveAllCallAliases(nodeId: NodeId, graph: DataflowGraph): Map edgeIncludesType(types, EdgeType.Reads | EdgeType.DefinedBy | EdgeType.DefinedByOnCall)) - .map(([t]) => [recoverContent(t, graph) ?? '', t] as const); + .map(([t]) => [recoverContent(t, graph) ?? '', t] as const) + .toArray(); /** only follow defined-by and reads */ queue = queue.concat(x); continue; @@ -152,7 +153,7 @@ function retrieveAllCallAliases(nodeId: NodeId, graph: DataflowGraph): Map edgeIncludesType(e.types, track) && (nodeId !== id || !edgeIncludesType(e.types, EdgeType.Argument))) .map(([t]) => t) ; @@ -212,9 +213,9 @@ function isParameterDefaultValue(nodeId: NodeId, ast: NormalizedAst): boolean { * This happens during the main resolution! * 4. Attach `linkTo` calls to the respective calls. */ -export async function executeCallContextQueries({ input }: BasicQueryData, queries: readonly CallContextQuery[]): Promise { - const ast = await input.normalize(); - const dataflow = await input.dataflow(); +export async function executeCallContextQueries({ analyzer }: BasicQueryData, queries: readonly CallContextQuery[]): Promise { + const dataflow = await analyzer.dataflow(); + const ast = await analyzer.normalize(); /* omit performance page load */ const now = Date.now(); @@ -226,7 +227,7 @@ export async function executeCallContextQueries({ input }: BasicQueryData, queri let cfg = undefined; if(requiresCfg) { - cfg = await input.controlflow([], true); + cfg = await analyzer.controlflow([], true); } const queriesWhichWantAliases = promotedQueries.filter(q => q.includeAliases); diff --git a/src/queries/catalog/cluster-query/cluster-query-executor.ts b/src/queries/catalog/cluster-query/cluster-query-executor.ts index 4007dc4596e..eacd913cc12 100644 --- a/src/queries/catalog/cluster-query/cluster-query-executor.ts +++ b/src/queries/catalog/cluster-query/cluster-query-executor.ts @@ -4,13 +4,13 @@ import { findAllClusters } from '../../../dataflow/cluster'; import type { BasicQueryData } from '../../base-query-format'; -export async function executeDataflowClusterQuery({ input }: BasicQueryData, queries: readonly DataflowClusterQuery[]): Promise { +export async function executeDataflowClusterQuery({ analyzer }: BasicQueryData, queries: readonly DataflowClusterQuery[]): Promise { if(queries.length !== 1) { log.warn('The dataflow cluster query expects only up to one query, but got', queries.length); } const start = Date.now(); - const clusters = findAllClusters((await input.dataflow()).graph); + const clusters = findAllClusters((await analyzer.dataflow()).graph); return { '.meta': { timing: Date.now() - start diff --git a/src/queries/catalog/config-query/config-query-executor.ts b/src/queries/catalog/config-query/config-query-executor.ts index a388cbbc4bc..6e1cf34842b 100644 --- a/src/queries/catalog/config-query/config-query-executor.ts +++ b/src/queries/catalog/config-query/config-query-executor.ts @@ -4,14 +4,14 @@ import type { BasicQueryData } from '../../base-query-format'; import { isNotUndefined } from '../../../util/assert'; import { deepMergeObjectInPlace } from '../../../util/objects'; -export function executeConfigQuery({ input }: BasicQueryData, queries: readonly ConfigQuery[]): Promise { +export function executeConfigQuery({ analyzer }: BasicQueryData, queries: readonly ConfigQuery[]): Promise { if(queries.length !== 1) { log.warn('Config query usually expects only up to one query, but got', queries.length); } const updates = queries.map(q => q.update).filter(isNotUndefined); for(const update of updates) { - deepMergeObjectInPlace(input.flowrConfig, update); + deepMergeObjectInPlace(analyzer.flowrConfig, update); } return Promise.resolve({ @@ -19,6 +19,6 @@ export function executeConfigQuery({ input }: BasicQueryData, queries: readonly /* there is no sense in measuring a get */ timing: 0 }, - config: input.flowrConfig + config: analyzer.flowrConfig }); } diff --git a/src/queries/catalog/control-flow-query/control-flow-query-executor.ts b/src/queries/catalog/control-flow-query/control-flow-query-executor.ts index 1cd40ad27d1..945d420b03a 100644 --- a/src/queries/catalog/control-flow-query/control-flow-query-executor.ts +++ b/src/queries/catalog/control-flow-query/control-flow-query-executor.ts @@ -3,7 +3,7 @@ import type { ControlFlowQuery, ControlFlowQueryResult } from './control-flow-qu import type { BasicQueryData } from '../../base-query-format'; -export async function executeControlFlowQuery({ input }: BasicQueryData, queries: readonly ControlFlowQuery[]): Promise { +export async function executeControlFlowQuery({ analyzer }: BasicQueryData, queries: readonly ControlFlowQuery[]): Promise { if(queries.length !== 1) { log.warn('The control flow query expects only up to one query, but got', queries.length); } @@ -11,7 +11,7 @@ export async function executeControlFlowQuery({ input }: BasicQueryData, queries const query = queries[0]; const start = Date.now(); - const controlFlow = await input.controlflow(query.config?.simplificationPasses, true); + const controlFlow = await analyzer.controlflow(query.config?.simplificationPasses, true); return { '.meta': { timing: Date.now() - start diff --git a/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-executor.ts b/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-executor.ts index bb3a7c03d96..8c377bd4875 100644 --- a/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-executor.ts +++ b/src/queries/catalog/dataflow-lens-query/dataflow-lens-query-executor.ts @@ -5,13 +5,13 @@ import { reduceDfg } from '../../../util/simple-df/dfg-view'; import { VertexType } from '../../../dataflow/graph/vertex'; -export async function executeDataflowLensQuery({ input }: BasicQueryData, queries: readonly DataflowLensQuery[]): Promise { +export async function executeDataflowLensQuery({ analyzer }: BasicQueryData, queries: readonly DataflowLensQuery[]): Promise { if(queries.length !== 1) { log.warn('Dataflow query expects only up to one query, but got', queries.length); } const now = Date.now(); - const simplifiedGraph = reduceDfg((await input.dataflow()).graph, { + const simplifiedGraph = reduceDfg((await analyzer.dataflow()).graph, { vertices: { keepEnv: false, keepCd: true, diff --git a/src/queries/catalog/dataflow-query/dataflow-query-executor.ts b/src/queries/catalog/dataflow-query/dataflow-query-executor.ts index 17168c5569f..d5e185a2814 100644 --- a/src/queries/catalog/dataflow-query/dataflow-query-executor.ts +++ b/src/queries/catalog/dataflow-query/dataflow-query-executor.ts @@ -3,7 +3,7 @@ import { log } from '../../../util/log'; import type { BasicQueryData } from '../../base-query-format'; -export async function executeDataflowQuery({ input }: BasicQueryData, queries: readonly DataflowQuery[]): Promise { +export async function executeDataflowQuery({ analyzer }: BasicQueryData, queries: readonly DataflowQuery[]): Promise { if(queries.length !== 1) { log.warn('Dataflow query expects only up to one query, but got', queries.length); } @@ -12,6 +12,6 @@ export async function executeDataflowQuery({ input }: BasicQueryData, queries: r /* there is no sense in measuring a get */ timing: 0 }, - graph: (await input.dataflow()).graph + graph: (await analyzer.dataflow()).graph }; } diff --git a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts index 9ead69ccc2e..735e15bcbbf 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts @@ -26,7 +26,7 @@ import { guard } from '../../../util/assert'; import type { NormalizedAst } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate'; export async function executeDependenciesQuery({ - input, + analyzer, libraries }: BasicQueryData, queries: readonly DependenciesQuery[]): Promise { if(queries.length !== 1) { @@ -34,13 +34,13 @@ export async function executeDependenciesQuery({ } const data = { - input, + input: analyzer, libraries: libraries }; - const normalize = await input.normalize(); - const dataflow = await input.dataflow(); - const config = input.flowrConfig; + const normalize = await analyzer.normalize(); + const dataflow = await analyzer.dataflow(); + const config = analyzer.flowrConfig; const now = Date.now(); const [query] = queries; diff --git a/src/queries/catalog/dependencies-query/dependencies-query-format.ts b/src/queries/catalog/dependencies-query/dependencies-query-format.ts index 5c126e1b368..53870e525b4 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-format.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-format.ts @@ -34,7 +34,7 @@ export const DefaultDependencyCategories = { /* for libraries, we have to additionally track all uses of `::` and `:::`, for this we currently simply traverse all uses */ additionalAnalysis: async(data, ignoreDefault, _functions, _queryResults, result) => { if(!ignoreDefault) { - visitAst((await data.input.normalize()).ast, n => { + visitAst((await data.analyzer.normalize()).ast, n => { if(n.type === RType.Symbol && n.namespace) { /* we should improve the identification of ':::' */ result.push({ diff --git a/src/queries/catalog/df-shape-query/df-shape-query-executor.ts b/src/queries/catalog/df-shape-query/df-shape-query-executor.ts index 4b52f646963..7c4ffa0c4e8 100644 --- a/src/queries/catalog/df-shape-query/df-shape-query-executor.ts +++ b/src/queries/catalog/df-shape-query/df-shape-query-executor.ts @@ -9,21 +9,21 @@ import type { SingleSlicingCriterion } from '../../../slicing/criterion/parse'; import { slicingCriterionToId } from '../../../slicing/criterion/parse'; import type { DataFrameDomain } from '../../../abstract-interpretation/data-frame/domain'; -export async function executeDfShapeQuery({ input }: BasicQueryData, queries: readonly DfShapeQuery[]): Promise { +export async function executeDfShapeQuery({ analyzer }: BasicQueryData, queries: readonly DfShapeQuery[]): Promise { if(queries.length !== 1 && queries.some(query => query.criterion === undefined)) { log.warn('The dataframe shape query expects only up to one query without slicing criterion, but got', queries.length); queries = [{ type: 'df-shape' }]; } - const ast = await input.normalize(); - const graph = (await input.dataflow()).graph; + const ast = await analyzer.normalize(); + const graph = (await analyzer.dataflow()).graph; const start = Date.now(); const domains = inferDataFrameShapes( - await input.controlflow(), + await analyzer.controlflow(), graph, ast, - input.flowrConfig); + analyzer.flowrConfig); if(queries.length === 1 && queries[0].criterion === undefined) { return { diff --git a/src/queries/catalog/happens-before-query/happens-before-query-executor.ts b/src/queries/catalog/happens-before-query/happens-before-query-executor.ts index 00020f7a2f6..683669e6f82 100644 --- a/src/queries/catalog/happens-before-query/happens-before-query-executor.ts +++ b/src/queries/catalog/happens-before-query/happens-before-query-executor.ts @@ -6,10 +6,10 @@ import { extractCfgQuick } from '../../../control-flow/extract-cfg'; import { happensBefore } from '../../../control-flow/happens-before'; import { slicingCriterionToId } from '../../../slicing/criterion/parse'; -export async function executeHappensBefore({ input }: BasicQueryData, queries: readonly HappensBeforeQuery[]): Promise { +export async function executeHappensBefore({ analyzer }: BasicQueryData, queries: readonly HappensBeforeQuery[]): Promise { const start = Date.now(); const results: Record = {}; - const ast = await input.normalize(); + const ast = await analyzer.normalize(); const cfg = extractCfgQuick(ast); for(const query of queries) { const { a, b } = query; diff --git a/src/queries/catalog/id-map-query/id-map-query-executor.ts b/src/queries/catalog/id-map-query/id-map-query-executor.ts index c44d1969285..6363722461d 100644 --- a/src/queries/catalog/id-map-query/id-map-query-executor.ts +++ b/src/queries/catalog/id-map-query/id-map-query-executor.ts @@ -2,7 +2,7 @@ import { log } from '../../../util/log'; import type { IdMapQuery, IdMapQueryResult } from './id-map-query-format'; import type { BasicQueryData } from '../../base-query-format'; -export async function executeIdMapQuery({ input }: BasicQueryData, queries: readonly IdMapQuery[]): Promise { +export async function executeIdMapQuery({ analyzer }: BasicQueryData, queries: readonly IdMapQuery[]): Promise { if(queries.length !== 1) { log.warn('Id-Map query expects only up to one query, but got', queries.length); } @@ -12,6 +12,6 @@ export async function executeIdMapQuery({ input }: BasicQueryData, queries: read /* there is no sense in measuring a get */ timing: 0 }, - idMap: (await input.normalize()).idMap + idMap: (await analyzer.normalize()).idMap }; } diff --git a/src/queries/catalog/lineage-query/lineage-query-executor.ts b/src/queries/catalog/lineage-query/lineage-query-executor.ts index b7fcf32b513..bd2c2e28ee3 100644 --- a/src/queries/catalog/lineage-query/lineage-query-executor.ts +++ b/src/queries/catalog/lineage-query/lineage-query-executor.ts @@ -3,14 +3,14 @@ import { log } from '../../../util/log'; import { getLineage } from '../../../cli/repl/commands/repl-lineage'; import type { BasicQueryData } from '../../base-query-format'; -export async function executeLineageQuery({ input }: BasicQueryData, queries: readonly LineageQuery[]): Promise { +export async function executeLineageQuery({ analyzer }: BasicQueryData, queries: readonly LineageQuery[]): Promise { const start = Date.now(); const result: LineageQueryResult['lineages'] = {}; for(const { criterion } of queries) { if(result[criterion]) { log.warn('Duplicate criterion in lineage query:', criterion); } - result[criterion] = getLineage(criterion, (await input.dataflow()).graph, (await input.normalize()).idMap); + result[criterion] = getLineage(criterion, (await analyzer.dataflow()).graph, (await analyzer.normalize()).idMap); } return { diff --git a/src/queries/catalog/linter-query/linter-query-executor.ts b/src/queries/catalog/linter-query/linter-query-executor.ts index 176fd6c5300..5ef9218b328 100644 --- a/src/queries/catalog/linter-query/linter-query-executor.ts +++ b/src/queries/catalog/linter-query/linter-query-executor.ts @@ -6,7 +6,7 @@ import { log } from '../../../util/log'; import type { ConfiguredLintingRule } from '../../../linter/linter-format'; import { executeLintingRule } from '../../../linter/linter-executor'; -export async function executeLinterQuery({ input }: BasicQueryData, queries: readonly LinterQuery[]): Promise { +export async function executeLinterQuery({ analyzer }: BasicQueryData, queries: readonly LinterQuery[]): Promise { const flattened = queries.flatMap(q => q.rules ?? (Object.keys(LintingRules) as LintingRuleNames[])); const distinct = new Set(flattened); if(distinct.size !== flattened.length) { @@ -20,7 +20,7 @@ export async function executeLinterQuery({ input }: BasicQueryData, queries: rea for(const entry of distinct) { const ruleName = typeof entry === 'string' ? entry : entry.name; - results.results[ruleName] = await executeLintingRule(ruleName, input, (entry as ConfiguredLintingRule)?.config); + results.results[ruleName] = await executeLintingRule(ruleName, analyzer, (entry as ConfiguredLintingRule)?.config); } return { diff --git a/src/queries/catalog/location-map-query/location-map-query-executor.ts b/src/queries/catalog/location-map-query/location-map-query-executor.ts index 163038fd514..c98d54c824d 100644 --- a/src/queries/catalog/location-map-query/location-map-query-executor.ts +++ b/src/queries/catalog/location-map-query/location-map-query-executor.ts @@ -23,7 +23,7 @@ function fuzzyFindFile(node: RNodeWithParent | undefined, idMap: AstIdMap): stri return ''; } -export async function executeLocationMapQuery({ input }: BasicQueryData, queries: readonly LocationMapQuery[]): Promise { +export async function executeLocationMapQuery({ analyzer }: BasicQueryData, queries: readonly LocationMapQuery[]): Promise { const start = Date.now(); const criteriaOfInterest = new Set(queries .flatMap(q => q.ids ?? []) @@ -36,13 +36,13 @@ export async function executeLocationMapQuery({ input }: BasicQueryData, queries }; let count = 0; const inverseMap = new Map(); - for(const file of (await input.dataflow()).graph.sourced) { + for(const file of (await analyzer.dataflow()).graph.sourced) { locationMap.files[count] = file; inverseMap.set(file, count); count++; } - const ast = await input.normalize(); + const ast = await analyzer.normalize(); for(const [id, node] of ast.idMap.entries()) { if(node.location && (criteriaOfInterest.size === 0 || criteriaOfInterest.has(id))) { const file = fuzzyFindFile(node, ast.idMap); diff --git a/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts b/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts index c37dd010ed9..28e42ecf5fb 100644 --- a/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts +++ b/src/queries/catalog/normalized-ast-query/normalized-ast-query-executor.ts @@ -3,7 +3,7 @@ import type { NormalizedAstQuery, NormalizedAstQueryResult } from './normalized- import type { BasicQueryData } from '../../base-query-format'; -export async function executeNormalizedAstQuery({ input }: BasicQueryData, queries: readonly NormalizedAstQuery[]): Promise { +export async function executeNormalizedAstQuery({ analyzer }: BasicQueryData, queries: readonly NormalizedAstQuery[]): Promise { if(queries.length !== 1) { log.warn('Normalized-Ast query expects only up to one query, but got', queries.length); } @@ -12,6 +12,6 @@ export async function executeNormalizedAstQuery({ input }: BasicQueryData, queri /* there is no sense in measuring a get */ timing: 0 }, - normalized: await input.normalize() + normalized: await analyzer.normalize() }; } diff --git a/src/queries/catalog/origin-query/origin-query-executor.ts b/src/queries/catalog/origin-query/origin-query-executor.ts index 809899e31f3..6c1d01192a4 100644 --- a/src/queries/catalog/origin-query/origin-query-executor.ts +++ b/src/queries/catalog/origin-query/origin-query-executor.ts @@ -9,7 +9,7 @@ export function fingerPrintOfQuery(query: OriginQuery): SingleSlicingCriterion { return query.criterion; } -export async function executeResolveValueQuery({ input }: BasicQueryData, queries: readonly OriginQuery[]): Promise { +export async function executeResolveValueQuery({ analyzer }: BasicQueryData, queries: readonly OriginQuery[]): Promise { const start = Date.now(); const results: OriginQueryResult['results'] = {}; for(const query of queries) { @@ -19,12 +19,12 @@ export async function executeResolveValueQuery({ input }: BasicQueryData, querie log.warn(`Duplicate Key for origin-query: ${key}, skipping...`); } - const astId = slicingCriterionToId(key, (await input.normalize()).idMap); + const astId = slicingCriterionToId(key, (await analyzer.normalize()).idMap); if(astId === undefined) { log.warn(`Could not resolve id for ${key}, skipping...`); continue; } - results[key] = getOriginInDfg((await input.dataflow()).graph, astId); + results[key] = getOriginInDfg((await analyzer.dataflow()).graph, astId); } return { diff --git a/src/queries/catalog/project-query/project-query-executor.ts b/src/queries/catalog/project-query/project-query-executor.ts index 0e116c43f03..87b81792dd3 100644 --- a/src/queries/catalog/project-query/project-query-executor.ts +++ b/src/queries/catalog/project-query/project-query-executor.ts @@ -2,7 +2,7 @@ import { log } from '../../../util/log'; import type { ProjectQuery, ProjectQueryResult } from './project-query-format'; import type { BasicQueryData } from '../../base-query-format'; -export async function executeProjectQuery({ input }: BasicQueryData, queries: readonly ProjectQuery[]): Promise { +export async function executeProjectQuery({ analyzer }: BasicQueryData, queries: readonly ProjectQuery[]): Promise { if(queries.length !== 1) { log.warn('Project query expects only up to one query, but got', queries.length); } @@ -11,6 +11,6 @@ export async function executeProjectQuery({ input }: BasicQueryData, queries: re /* there is no sense in measuring a get */ timing: 0 }, - files: (await input.dataflow()).graph.sourced + files: (await analyzer.dataflow()).graph.sourced }; } diff --git a/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts b/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts index e927da55d58..151255ad0a9 100644 --- a/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts +++ b/src/queries/catalog/resolve-value-query/resolve-value-query-executor.ts @@ -8,12 +8,12 @@ export function fingerPrintOfQuery(query: ResolveValueQuery): string { return JSON.stringify(query); } -export async function executeResolveValueQuery({ input }: BasicQueryData, queries: readonly ResolveValueQuery[]): Promise { +export async function executeResolveValueQuery({ analyzer }: BasicQueryData, queries: readonly ResolveValueQuery[]): Promise { const start = Date.now(); const results: ResolveValueQueryResult['results'] = {}; - const graph = (await input.dataflow()).graph; - const ast = await input.normalize(); + const graph = (await analyzer.dataflow()).graph; + const ast = await analyzer.normalize(); for(const query of queries) { const key = fingerPrintOfQuery(query); @@ -24,7 +24,7 @@ export async function executeResolveValueQuery({ input }: BasicQueryData, querie const values = query.criteria .map(criteria => slicingCriterionToId(criteria, ast.idMap)) - .flatMap(ident => resolveIdToValue(ident, { graph, full: true, idMap: ast.idMap, resolve: input.flowrConfig.solver.variables })); + .flatMap(ident => resolveIdToValue(ident, { graph, full: true, idMap: ast.idMap, resolve: analyzer.flowrConfig.solver.variables })); results[key] = { values: values diff --git a/src/queries/catalog/search-query/search-query-executor.ts b/src/queries/catalog/search-query/search-query-executor.ts index d9dbea0401c..a4cdb205a09 100644 --- a/src/queries/catalog/search-query/search-query-executor.ts +++ b/src/queries/catalog/search-query/search-query-executor.ts @@ -4,13 +4,13 @@ import { runSearch } from '../../../search/flowr-search-executor'; import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id'; import type { FlowrSearch } from '../../../search/flowr-search-builder'; -export async function executeSearch({ input }: BasicQueryData, queries: readonly SearchQuery[]): Promise { +export async function executeSearch({ analyzer }: BasicQueryData, queries: readonly SearchQuery[]): Promise { const start = Date.now(); const results: { ids: NodeId[], search: FlowrSearch }[] = []; for(const query of queries) { const { search } = query; - const searchResult = await runSearch(search, input); + const searchResult = await runSearch(search, analyzer); results.push({ ids: searchResult.getElements().map(({ node }) => node.info.id), diff --git a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts index 9c310c0d185..9e4a22fe4fe 100644 --- a/src/queries/catalog/static-slice-query/static-slice-query-executor.ts +++ b/src/queries/catalog/static-slice-query/static-slice-query-executor.ts @@ -11,7 +11,7 @@ export function fingerPrintOfQuery(query: StaticSliceQuery): string { return JSON.stringify(query); } -export async function executeStaticSliceQuery({ input }: BasicQueryData, queries: readonly StaticSliceQuery[]): Promise { +export async function executeStaticSliceQuery({ analyzer }: BasicQueryData, queries: readonly StaticSliceQuery[]): Promise { const start = Date.now(); const results: StaticSliceQueryResult['results'] = {}; for(const query of queries) { @@ -21,13 +21,13 @@ export async function executeStaticSliceQuery({ input }: BasicQueryData, queries } const { criteria, noReconstruction, noMagicComments } = query; const sliceStart = Date.now(); - const slice = staticSlice(await input.dataflow(), await input.normalize(), criteria, query.direction ?? SliceDirection.Backward, input.flowrConfig.solver.slicer?.threshold); + const slice = staticSlice(await analyzer.dataflow(), await analyzer.normalize(), criteria, query.direction ?? SliceDirection.Backward, analyzer.flowrConfig.solver.slicer?.threshold); const sliceEnd = Date.now(); if(noReconstruction) { results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart } } }; } else { const reconstructStart = Date.now(); - const reconstruct = reconstructToCode(await input.normalize(), slice.result, noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)); + const reconstruct = reconstructToCode(await analyzer.normalize(), slice.result, noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)); const reconstructEnd = Date.now(); results[key] = { slice: { ...slice, '.meta': { timing: sliceEnd - sliceStart } }, diff --git a/src/search/search-executor/search-generators.ts b/src/search/search-executor/search-generators.ts index 4c26a2134b3..29fd89706ad 100644 --- a/src/search/search-executor/search-generators.ts +++ b/src/search/search-executor/search-generators.ts @@ -90,7 +90,7 @@ function generateFrom(_input: FlowrAnalysisProvider, args: { from: FlowrSearchEl async function generateFromQuery(input: FlowrAnalysisProvider, args: { from: readonly Query[] }): Promise[]>> { - const result = await executeQueries({ input }, args.from); + const result = await executeQueries({ analyzer: input }, args.from); // collect involved nodes const nodesByQuery = new Map>>(); diff --git a/test/functionality/_helper/query.ts b/test/functionality/_helper/query.ts index 10a210a1b0e..49a8f478219 100644 --- a/test/functionality/_helper/query.ts +++ b/test/functionality/_helper/query.ts @@ -81,7 +81,7 @@ export function assertQuery< const dummyProject = getDummyFlowrProject(); const result = await executeQueries({ - input: analyzer, + analyzer: analyzer, libraries: dummyProject.libraries }, queries); From c413dcbfbb2e1c4dce2084f43f434a05d542653f Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Fri, 26 Sep 2025 01:28:42 +0200 Subject: [PATCH 67/70] feat-fix: data naming --- .../dependencies-query/dependencies-query-executor.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts index 735e15bcbbf..e95b0d3bab9 100644 --- a/src/queries/catalog/dependencies-query/dependencies-query-executor.ts +++ b/src/queries/catalog/dependencies-query/dependencies-query-executor.ts @@ -34,8 +34,8 @@ export async function executeDependenciesQuery({ } const data = { - input: analyzer, - libraries: libraries + analyzer, + libraries }; const normalize = await analyzer.normalize(); @@ -60,7 +60,7 @@ export async function executeDependenciesQuery({ const results = Object.fromEntries(functions.entries().map(([c, f]) => { const results = getResults(queries, { dataflow, config, normalize }, queryResults, c, f, data); - // only default categories allow additional analyses, so we null coalese here! + // only default categories allow additional analyses, so we null-coalesce here! (DefaultDependencyCategories as Record)[c]?.additionalAnalysis?.(data, ignoreDefault, f, queryResults, results); return [c, results]; })) as {[C in DependencyCategoryName]?: DependencyInfo[]}; From 2bf0d093b7d0932fe798713ea5bb1618b042c491 Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Fri, 26 Sep 2025 01:30:27 +0200 Subject: [PATCH 68/70] lint-fix: handle linter awaits --- .../functionality/project/plugin/description-file.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/functionality/project/plugin/description-file.test.ts b/test/functionality/project/plugin/description-file.test.ts index bff7b685dc2..d846f3fb612 100644 --- a/test/functionality/project/plugin/description-file.test.ts +++ b/test/functionality/project/plugin/description-file.test.ts @@ -17,20 +17,20 @@ describe('DESCRIPTION-file', function() { const descriptionFilePlugin = new FlowrAnalyzerDescriptionFilePlugin(); descriptionFilePlugin.addFiles(path.resolve('test/testfiles/project/DESCRIPTION')); describe.sequential('Parsing', function() { - test('Library-Versions-Plugin', async() => { + test('Library-Versions-Plugin', () => { const flowrAnalyzerPackageVersionsDescriptionFilePlugin = new FlowrAnalyzerPackageVersionsDescriptionFilePlugin(); flowrAnalyzerPackageVersionsDescriptionFilePlugin.dependencies = [descriptionFilePlugin]; - await flowrAnalyzerPackageVersionsDescriptionFilePlugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); + flowrAnalyzerPackageVersionsDescriptionFilePlugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); assert.isNotEmpty(flowrAnalyzerPackageVersionsDescriptionFilePlugin.packages); }); - test('Loading-Order-Plugin', async() => { + test('Loading-Order-Plugin', () => { const flowrAnalyzerLoadingOrderDescriptionFilePlugin = new FlowrAnalyzerLoadingOrderDescriptionFilePlugin(); flowrAnalyzerLoadingOrderDescriptionFilePlugin.dependencies = [descriptionFilePlugin]; - await flowrAnalyzerLoadingOrderDescriptionFilePlugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); + flowrAnalyzerLoadingOrderDescriptionFilePlugin.processor({} as FlowrAnalyzer, {} as FlowrConfigOptions); assert.isNotEmpty(flowrAnalyzerLoadingOrderDescriptionFilePlugin.loadingOrder); }); From 038d1a3da3092970079a20cd1e044fa52aa36f55 Mon Sep 17 00:00:00 2001 From: Florian Sihler Date: Fri, 26 Sep 2025 01:34:07 +0200 Subject: [PATCH 69/70] refactor: remove only debug fixture --- .../control-flow/control-flow-graph.test.ts | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/test/functionality/control-flow/control-flow-graph.test.ts b/test/functionality/control-flow/control-flow-graph.test.ts index 6558ee9cbd5..1003c697ea9 100644 --- a/test/functionality/control-flow/control-flow-graph.test.ts +++ b/test/functionality/control-flow/control-flow-graph.test.ts @@ -426,29 +426,27 @@ describe('Control Flow Graph', withTreeSitter(parser => { }, { withBasicBlocks: true }); - describe.only('conditionals', () => { - assertCfg(parser, 'f <- function(x) x\nf()', { - entryPoints: ['bb-5'], - exitPoints: ['bb-9-exit'], - graph: new ControlFlowGraph() - .addVertex({ - id: 'bb-8-exit', - type: CfgVertexType.Block, - elems: [ - { id: '8-exit', type: CfgVertexType.EndMarker, root: 8 }, - { id: 7, type: CfgVertexType.Expression }, - { - id: 8, - type: CfgVertexType.Statement, - mid: [0], - end: ['8-exit'], - callTargets: new Set([5]) - }, - { id: '6-exit', type: CfgVertexType.EndMarker, root: 6 } - - ] - }) - }, { expectIsSubgraph: true, withBasicBlocks: true }); - }); + assertCfg(parser, 'f <- function(x) x\nf()', { + entryPoints: ['bb-5'], + exitPoints: ['bb-9-exit'], + graph: new ControlFlowGraph() + .addVertex({ + id: 'bb-8-exit', + type: CfgVertexType.Block, + elems: [ + { id: '8-exit', type: CfgVertexType.EndMarker, root: 8 }, + { id: 7, type: CfgVertexType.Expression }, + { + id: 8, + type: CfgVertexType.Statement, + mid: [0], + end: ['8-exit'], + callTargets: new Set([5]) + }, + { id: '6-exit', type: CfgVertexType.EndMarker, root: 6 } + + ] + }) + }, { expectIsSubgraph: true, withBasicBlocks: true }); }); })); From b6d7a0b43774848a9be9187021d27b8201d06d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=2E=20Sch=C3=B6ller?= <7847075+MaxAtoms@users.noreply.github.com> Date: Fri, 26 Sep 2025 02:09:02 +0200 Subject: [PATCH 70/70] Add analyzer to Wiki (#1940) * docs(analyzer): add analyzer to wiki * docs(analyzer): use analyzer for ast and dataflow * docs(analyzer): add query api example * docs(analyzer): fix line breaks * feat-fix(analyzer): kill open engines --------- Co-authored-by: Florian Sihler --- src/documentation/print-core-wiki.ts | 43 ++++++++++++++++-- .../print-dataflow-graph-wiki.ts | 45 ++++++++----------- src/documentation/print-interface-wiki.ts | 31 +++++++++---- .../print-normalized-ast-wiki.ts | 16 +++---- src/engines.ts | 10 +++-- src/project/flowr-analyzer-builder.ts | 4 +- src/project/flowr-analyzer.ts | 8 ++++ 7 files changed, 105 insertions(+), 52 deletions(-) diff --git a/src/documentation/print-core-wiki.ts b/src/documentation/print-core-wiki.ts index 01df7cd65d8..82295a5ee3a 100644 --- a/src/documentation/print-core-wiki.ts +++ b/src/documentation/print-core-wiki.ts @@ -43,6 +43,8 @@ import { PipelineExecutor } from '../core/pipeline-executor'; import { createPipeline } from '../core/steps/pipeline/pipeline'; import { staticSlice } from '../slicing/static/static-slicer'; import { defaultConfigOptions } from '../config'; +import { FlowrAnalyzerBuilder } from '../project/flowr-analyzer-builder'; +import { FlowrAnalyzer } from '../project/flowr-analyzer'; async function getText(shell: RShell) { const rversion = (await shell.usedRVersion())?.format() ?? 'unknown'; @@ -83,6 +85,7 @@ See the [Getting flowR to Talk](#getting-flowr-to-talk) section below for more i ` })} +* [Creating and Using a flowR Analyzer Instance](#creating-and-using-a-flowr-analyzer-instance) * [Pipelines and their Execution](#pipelines-and-their-execution) * [How flowR Produces Dataflow Graphs](#how-flowr-produces-dataflow-graphs) * [Overview](#overview) @@ -92,10 +95,45 @@ See the [Getting flowR to Talk](#getting-flowr-to-talk) section below for more i * [Beyond the Dataflow Graph](#beyond-the-dataflow-graph) * [Static Backward Slicing](#static-backward-slicing) * [Getting flowR to Talk](#getting-flowr-to-talk) + +## Creating and Using a flowR Analyzer Instance + +The ${shortLink(FlowrAnalyzerBuilder.name, info)} class should be used as a starting point to create analyses in _flowR_. +It provides a fluent interface for the configuration and creation of a ${shortLink(FlowrAnalyzer.name, info)} instance: + +${codeBlock('typescript', ` +const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('x <- 1; y <- x; print(y);')) + .amendConfig(c => { + c.ignoreSourceCalls = true; + }) + .setEngine('r-shell') + .build(); +`)} + +${shortLink(requestFromInput.name, info)} is merely a convenience function to create a request object from a code string. + +The analyzer instance can then be used to access analysis results like the normalized AST, the dataflow graph, and the controlflow graph: + +${codeBlock('typescript', ` +const normalizedAst = await analyzer.normalizedAst(); +const dataflow = await analyzer.dataflow(); +const cfg = await analyzer.controlFlow(); +`)} + +The analyzer also exposes the [query API](${FlowrWikiBaseRef}/Query-API): + +${codeBlock('typescript', ` +const result = await analyzer.query([ + { + type: 'static-slice', + criteria: ['1@y'] + } +]); +`)} ## Pipelines and their Execution -At the core of every analysis by flowR is the ${shortLink(PipelineExecutor.name, info)} class which takes a sequence of analysis steps (in the form of a ${shortLink('Pipeline', info)}) and executes it +At the core of every analysis done via a ${shortLink(FlowrAnalyzer.name, info)} is the ${shortLink(PipelineExecutor.name, info)} class which takes a sequence of analysis steps (in the form of a ${shortLink('Pipeline', info)}) and executes it on a given input. In general, these pipeline steps are analysis agnostic and may use arbitrary input and ordering. However, two important and predefined pipelines, the ${shortLink('DEFAULT_DATAFLOW_PIPELINE', info)} and the ${shortLink('TREE_SITTER_DATAFLOW_PIPELINE', info)} adequately cover the most common analysis steps (differentiated only by the [Engine](${FlowrWikiBaseRef}/Engines) used). @@ -118,10 +156,9 @@ const executor = new PipelineExecutor(TREE_SITTER_DATAFLOW_PIPELINE, { const result = await executor.allRemainingSteps(); `)} -This is, roughly, what the ${shortLink('replGetDataflow', info)} function does for the ${getReplCommand('dataflow')} REPL command when using the [\`tree-sitter\` engine](${FlowrWikiBaseRef}/Engines). +This is, roughly, what the ${shortLink('dataflow', info)} function does when using the [\`tree-sitter\` engine](${FlowrWikiBaseRef}/Engines). We create a new ${shortLink(PipelineExecutor.name, info)} with the ${shortLink('TREE_SITTER_DATAFLOW_PIPELINE', info)} and then use ${shortLink(`${PipelineExecutor.name}::${new PipelineExecutor(TREE_SITTER_PARSE_PIPELINE, { parser: new TreeSitterExecutor(), request: requestFromInput('') }, defaultConfigOptions).allRemainingSteps.name}`, info)} to cause the execution of all contained steps (in general, pipelines can be executed step-by-step, but this is usually not required if you just want the result). -${shortLink(requestFromInput.name, info)} is merely a convenience function to create a request object from a code string. In general, however, most flowR-internal functions which are tasked with generating dataflow prefer the use of ${shortLink(createDataflowPipeline.name, info)} as this function automatically selects the correct pipeline based on the engine used. diff --git a/src/documentation/print-dataflow-graph-wiki.ts b/src/documentation/print-dataflow-graph-wiki.ts index 4ede30ae801..45ce5e2d7dd 100644 --- a/src/documentation/print-dataflow-graph-wiki.ts +++ b/src/documentation/print-dataflow-graph-wiki.ts @@ -8,7 +8,6 @@ import { DataflowGraphBuilder, emptyGraph } from '../dataflow/graph/dataflowgrap import { guard } from '../util/assert'; import { formatSideEffect, printDfGraph, printDfGraphForCode, verifyExpectedSubgraph } from './doc-util/doc-dfg'; import { FlowrGithubBaseRef, FlowrWikiBaseRef, getFilePathMd } from './doc-util/doc-files'; -import { PipelineExecutor } from '../core/pipeline-executor'; import { requestFromInput } from '../r-bridge/retriever'; import { jsonReplacer } from '../util/json'; import { printEnvironmentToMarkdown } from './doc-util/doc-env'; @@ -26,8 +25,7 @@ import { recoverContent, recoverName } from '../r-bridge/lang-4.x/ast/model/proc import { ReferenceType } from '../dataflow/environments/identifier'; import { EmptyArgument } from '../r-bridge/lang-4.x/ast/model/nodes/r-function-call'; import { resolveByName, resolvesToBuiltInConstant, } from '../dataflow/environments/resolve-by-name'; -import { createDataflowPipeline, DEFAULT_DATAFLOW_PIPELINE } from '../core/steps/pipeline/default-pipelines'; -import type { PipelineOutput } from '../core/steps/pipeline/pipeline'; +import { createDataflowPipeline } from '../core/steps/pipeline/default-pipelines'; import { autoGenHeader } from './doc-util/doc-auto-gen'; import { nth } from '../util/text/text'; import { setMinLevelOfAllLogs } from '../../test/functionality/_helper/log'; @@ -45,6 +43,8 @@ import { } from '../dataflow/internal/process/functions/call/unnamed-call-handling'; import { defaultEnv } from '../../test/functionality/_helper/dataflow/environment-builder'; import { defaultConfigOptions } from '../config'; +import { FlowrAnalyzerBuilder } from '../project/flowr-analyzer-builder'; +import type { DataflowInformation } from '../dataflow/info'; async function subExplanation(shell: RShell, { description, code, expectedSubgraph }: SubExplanationParameters): Promise { expectedSubgraph = await verifyExpectedSubgraph(shell, code, expectedSubgraph); @@ -862,13 +862,10 @@ ${details('Example: While-Loop Body', await printDfGraphForCode(shell, 'while(TR return results.join('\n'); } -async function dummyDataflow(): Promise> { - const shell = new RShell(); - const result = await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, { - parser: shell, - request: requestFromInput('x <- 1\nx + 1') - }, defaultConfigOptions).allRemainingSteps(); - shell.close(); +async function dummyDataflow(): Promise { + const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('x <- 1\nx + 1')).build(); + const result = await analyzer.dataflow(); + analyzer.close(); return result; } @@ -1009,16 +1006,12 @@ ${details('Example: Nested Conditionals', await printDfGraphForCode(shell, 'if(x ${section('Dataflow Information', 2, 'dataflow-information')} -Using _flowR's_ code interface (see the [Interface](${FlowrWikiBaseRef}/Interface) wiki page for more), you can generate the dataflow information -for a given piece of R code (in this case \`x <- 1; x + 1\`) as follows (using the ${shortLink(RShell.name, vertexType.info)} and the ${shortLink(PipelineExecutor.name, vertexType.info)} classes): +Using _flowR's_ code interface (see the [Interface](${FlowrWikiBaseRef}/Interface#creating-flowr-analyses) wiki page for more), you can generate the dataflow information +for a given piece of R code (in this case \`x <- 1; x + 1\`) as follows: ${codeBlock('ts', ` -const shell = new ${RShell.name}() -const result = await new ${PipelineExecutor.name}(DEFAULT_DATAFLOW_PIPELINE, { - shell, - request: ${requestFromInput.name}('x <- 1; x + 1') -}).allRemainingSteps(); -shell.close(); +const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('x <- 1\nx + 1')).build(); +const result = await analyzer.dataflow(); `)}
@@ -1037,7 +1030,7 @@ Now, you can find the dataflow _information_ with \`result.dataflow\`. More spec ${ await (async() => { const result = await dummyDataflow(); - const dfGraphString = printDfGraph(result.dataflow.graph); + const dfGraphString = printDfGraph(result.graph); return ` ${dfGraphString} @@ -1049,7 +1042,7 @@ However, the dataflow information contains more, quite a lot of information in f Dataflow Information as Json _As the information is pretty long, we inhibit pretty printing and syntax highlighting:_ -${codeBlock('text', JSON.stringify(result.dataflow, jsonReplacer))} +${codeBlock('text', JSON.stringify(result, jsonReplacer))}
@@ -1059,15 +1052,15 @@ ${ printHierarchy({ program: vertexType.program, info: vertexType.info, root: 'DataflowInformation' }) } -Let's start by looking at the properties of the dataflow information object: ${Object.keys(result.dataflow).map(k => `\`${k}\``).join(', ')}. +Let's start by looking at the properties of the dataflow information object: ${Object.keys(result).map(k => `\`${k}\``).join(', ')}. ${ (() => { /* this includes the meta field for timing */ - guard(Object.keys(result.dataflow).length === 8, () => 'Update Dataflow Documentation!'); return ''; + guard(Object.keys(result).length === 8, () => 'Update Dataflow Documentation!'); return ''; })() } There are three sets of references. -**in** (ids: ${JSON.stringify(new Set(result.dataflow.in.map(n => n.nodeId)), jsonReplacer)}) and **out** (ids: ${JSON.stringify(new Set(result.dataflow.out.map(n => n.nodeId)), jsonReplacer)}) contain the +**in** (ids: ${JSON.stringify(new Set(result.in.map(n => n.nodeId)), jsonReplacer)}) and **out** (ids: ${JSON.stringify(new Set(result.out.map(n => n.nodeId)), jsonReplacer)}) contain the ingoing and outgoing references of the subgraph at hand (in this case, the whole code, as we are at the end of the dataflow analysis). Besides the Ids, they also contain important meta-information (e.g., what is to be read). The third set, **unknownReferences**, contains all references that are not yet identified as read or written @@ -1077,15 +1070,15 @@ The **environment** property contains the active environment information of the In other words, this is a linked list of tables (scopes), mapping identifiers to their respective definitions. A summarized version of the produced environment looks like this: -${printEnvironmentToMarkdown(result.dataflow.environment.current)} +${printEnvironmentToMarkdown(result.environment.current)} This shows us that the local environment contains a single definition for \`x\` (with id 0) and that the parent environment is the built-in environment. Additionally, we get the information that the node with the id 2 was responsible for the definition of \`x\`. Last but not least, the information contains the single **entry point** (${ - JSON.stringify(result.dataflow.entryPoint) + JSON.stringify(result.entryPoint) }) and a set of **exit points** (${ - JSON.stringify(result.dataflow.exitPoints.map(e => e.nodeId)) + JSON.stringify(result.exitPoints.map(e => e.nodeId)) }). Besides marking potential exits, the exit points also provide information about why the exit occurs and which control dependencies affect the exit. diff --git a/src/documentation/print-interface-wiki.ts b/src/documentation/print-interface-wiki.ts index 84a31ec7782..df80a34128b 100644 --- a/src/documentation/print-interface-wiki.ts +++ b/src/documentation/print-interface-wiki.ts @@ -12,14 +12,8 @@ import { fileProtocol, removeRQuotes, requestFromInput } from '../r-bridge/retri import { DockerName } from './doc-util/doc-docker'; import { documentReplSession, printReplHelpAsMarkdownTable } from './doc-util/doc-repl'; import { printDfGraphForCode } from './doc-util/doc-dfg'; -import type { - FlowrConfigOptions } from '../config'; -import { - DropPathsOption, - flowrConfigFileSchema, - InferWorkingDirectory, - VariableResolve -} from '../config'; +import type { FlowrConfigOptions } from '../config'; +import { DropPathsOption, flowrConfigFileSchema, InferWorkingDirectory, VariableResolve } from '../config'; import { describeSchema } from '../util/schema'; import { markdownFormatter } from '../util/text/ansi'; import { defaultConfigFile } from '../cli/flowr-main-options'; @@ -29,6 +23,7 @@ import { block, details } from './doc-util/doc-structure'; import { getTypesFromFolder, mermaidHide, shortLink } from './doc-util/doc-types'; import path from 'path'; import { TreeSitterExecutor } from '../r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; +import { FlowrAnalyzer } from '../project/flowr-analyzer'; async function explainServer(shell: RShell): Promise { documentAllServerMessages(); @@ -313,13 +308,31 @@ ${shortLink(RShell.name + '::' + shell.sendCommandWithOutput.name, types.info, t Besides that, the command ${shortLink(RShell.name + '::' + shell.tryToInjectHomeLibPath.name, types.info)} may be of interest, as it enables all libraries available on the host system. +### Creating _flowR_ analyses + +Nowadays, instances of ${shortLink(FlowrAnalyzer.name, types.info)} should be used as central frontend to get analysis results from _flowR_. +For example, a program slice can be created like this: + +${ + codeBlock('ts', ` +const analyzer = await new FlowrAnalyzerBuilder(requestFromInput('x <- 1\\ny <- x\\nx')).build(); +const result = await analyzer.query([ + { + type: 'static-slice', + criteria: ['3@x'] + } +]); +//console.log(result['static-slice']); +`) +} + ### The Pipeline Executor Once, in the beginning, _flowR_ was meant to produce a dataflow graph merely to provide *program slices*. However, with continuous updates, the [dataflow graph](${FlowrWikiBaseRef}/Dataflow%20Graph) repeatedly proves to be the more interesting part. With this, we restructured _flowR_'s originally *hardcoded* pipeline to be far more flexible. Now, it can be theoretically extended or replaced with arbitrary steps, optional steps, and what we call 'decorations' of these steps. -In short, if you still "just want to slice" you can do it like this with the ${shortLink(PipelineExecutor.name, types.info)}: +In short, a slicing pipeline using the ${shortLink(PipelineExecutor.name, types.info)} looks like this: ${ codeBlock('ts', ` diff --git a/src/documentation/print-normalized-ast-wiki.ts b/src/documentation/print-normalized-ast-wiki.ts index eafffc299d8..321bee0b029 100644 --- a/src/documentation/print-normalized-ast-wiki.ts +++ b/src/documentation/print-normalized-ast-wiki.ts @@ -4,7 +4,7 @@ import { LogLevel } from '../util/log'; import { autoGenHeader } from './doc-util/doc-auto-gen'; import { codeBlock } from './doc-util/doc-code'; import { printNormalizedAstForCode } from './doc-util/doc-normalized-ast'; -import { mermaidHide, printHierarchy, getTypesFromFolder, shortLink } from './doc-util/doc-types'; +import { getTypesFromFolder, mermaidHide, printHierarchy, shortLink } from './doc-util/doc-types'; import path from 'path'; import { FlowrGithubBaseRef, FlowrWikiBaseRef, getFilePathMd } from './doc-util/doc-files'; import { getReplCommand } from './doc-util/doc-cli-option'; @@ -16,6 +16,7 @@ import { visitAst } from '../r-bridge/lang-4.x/ast/model/processing/visitor'; import { collectAllIds } from '../r-bridge/lang-4.x/ast/model/collect'; import { DefaultNormalizedAstFold } from '../abstract-interpretation/normalized-ast-fold'; import { createNormalizePipeline } from '../core/steps/pipeline/default-pipelines'; +import { FlowrAnalyzer } from '../project/flowr-analyzer'; async function getText(shell: RShell) { const rversion = (await shell.usedRVersion())?.format() ?? 'unknown'; @@ -92,17 +93,14 @@ The following segments intend to give you an overview of how to work with the no ## How to Get a Normalized AST -As explained alongside the [Interface](${FlowrWikiBaseRef}/Interface#the-pipeline-executor) wiki page, you can use the -${shortLink(PipelineExecutor.name, types.info)} to get the ${shortLink('NormalizedAst', types.info)}. If you are only interested in the normalization, -a pipeline like the ${shortLink('DEFAULT_NORMALIZE_PIPELINE', types.info)} suffices: +As explained alongside the [Interface](${FlowrWikiBaseRef}/Interface#creating-flowr-analyses) wiki page, you can use an instance of +${shortLink(FlowrAnalyzer.name, types.info)} to get the ${shortLink('NormalizedAst', types.info)}: ${codeBlock('ts', ` async function getAst(code: string): Promise { - const result = await new ${PipelineExecutor.name}(DEFAULT_NORMALIZE_PIPELINE, { - parser: new ${RShell.name}(), - request: ${requestFromInput.name}(code.trim()) - }).allRemainingSteps(); - return result.normalize.ast; + const analyzer = await new FlowrAnalyzerBuilder(${requestFromInput.name}(code.trim())).build(); + const result = analyzer.normalizedAst(); + return result.ast; }`)} From the REPL, you can use the ${getReplCommand('normalize')} command. diff --git a/src/engines.ts b/src/engines.ts index 7d87f734ca8..ad17c3b7fb9 100644 --- a/src/engines.ts +++ b/src/engines.ts @@ -5,9 +5,13 @@ import { bold, ColorEffect, Colors, formatter, italic } from './util/text/ansi'; import { TreeSitterExecutor } from './r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; import { log } from './util/log'; -export async function retrieveEngineInstances(config: FlowrConfigOptions): Promise<{ engines: KnownEngines, default: keyof KnownEngines }> { +/** + * Retrieve all requested engine instance. + * Please make sure that if this includes the R engine, that you properly shut it down again! + */ +export async function retrieveEngineInstances(config: FlowrConfigOptions, defaultOnly = false): Promise<{ engines: KnownEngines, default: keyof KnownEngines }> { const engines: KnownEngines = {}; - if(getEngineConfig(config, 'r-shell')) { + if(getEngineConfig(config, 'r-shell') && (!defaultOnly || config.defaultEngine === 'r-shell')) { // we keep an active shell session to allow other parse investigations :) engines['r-shell'] = new RShell(getEngineConfig(config, 'r-shell'), { revive: RShellReviveOptions.Always, @@ -18,7 +22,7 @@ export async function retrieveEngineInstances(config: FlowrConfigOptions): Promi } }); } - if(getEngineConfig(config, 'tree-sitter')) { + if(getEngineConfig(config, 'tree-sitter') && (!defaultOnly || config.defaultEngine === 'tree-sitter')) { await TreeSitterExecutor.initTreeSitter(getEngineConfig(config, 'tree-sitter')); engines['tree-sitter'] = new TreeSitterExecutor(); } diff --git a/src/project/flowr-analyzer-builder.ts b/src/project/flowr-analyzer-builder.ts index 893b0fa6788..0990c01fa2b 100644 --- a/src/project/flowr-analyzer-builder.ts +++ b/src/project/flowr-analyzer-builder.ts @@ -116,8 +116,8 @@ export class FlowrAnalyzerBuilder { if(this.parser) { parser = this.parser; } else { - const engines = await retrieveEngineInstances(this.flowrConfig); - parser = this.parser ?? engines.engines[engines.default] as KnownParser; + const engines = await retrieveEngineInstances(this.flowrConfig, true); + parser = engines.engines[engines.default] as KnownParser; } guard(this.request !== undefined, 'Currently we require at least one request to build an analyzer, please provide one using the constructor or the addRequest method'); diff --git a/src/project/flowr-analyzer.ts b/src/project/flowr-analyzer.ts index ca4aa2902b6..610c70bed04 100644 --- a/src/project/flowr-analyzer.ts +++ b/src/project/flowr-analyzer.ts @@ -133,4 +133,12 @@ export class FlowrAnalyzer { >(search: Search): Promise>> { return runSearch(search, this); } + + + /** + * Close the parser if it was created by this builder. This is only required if you rely on an RShell/remote engine. + */ + public close() { + return this.parser?.close(); + } } \ No newline at end of file