Add type inference

package.json

@@ -42,8 +42,9 @@
     "gen:linter-issue": "ts-node src/documentation/print-linter-issue.ts",
     "build": "tsc --project .",
     "build-dev": "npm run build && npm run build:copy-wasm",
-    "build:bundle-flowr": "npm run build && esbuild --bundle dist/src/cli/flowr.js --platform=node --tree-shaking=true --bundle --minify --external:clipboardy --target=node22 --outfile=dist/src/cli/flowr.min.js && npm run build:copy-wasm",
+    "build:bundle-flowr": "npm run build && esbuild --bundle dist/src/cli/flowr.js --platform=node --tree-shaking=true --bundle --minify --external:clipboardy --target=node22 --outfile=dist/src/cli/flowr.min.js && npm run build:copy-wasm && npm run build:copy-csvs",
     "build:copy-wasm": "mkdir -p dist/node_modules/@eagleoutice/tree-sitter-r/ && mkdir -p dist/node_modules/web-tree-sitter && cp node_modules/@eagleoutice/tree-sitter-r/tree-sitter-r.wasm dist/node_modules/@eagleoutice/tree-sitter-r/  && cp node_modules/web-tree-sitter/tree-sitter.wasm dist/node_modules/web-tree-sitter/",
+    "build:copy-csvs": "cp src/typing/adapter/traced-function-types.csv dist/src/typing/adapter/traced-function-types.csv && cp src/typing/adapter/turcotte-types.csv dist/src/typing/adapter/turcotte-types.csv",
     "lint-local": "npx eslint --version && npx eslint src/ test/ --rule \"no-warning-comments: off\"",
     "lint": "npm run license-compat -- --summary && npx eslint --version && npx eslint src/ test/",
     "license-compat": "license-checker --onlyAllow 'MIT;MIT OR X11;GPLv2;LGPL;GNUGPL;ISC;Apache-2.0;FreeBSD;BSD-2-Clause;clearbsd;ModifiedBSD;BSD-3-Clause;Python-2.0;Unlicense;WTFPL;BlueOak-1.0.0;CC-BY-4.0;CC-BY-3.0;CC0-1.0;0BSD'",
@@ -212,6 +213,7 @@
     "clipboardy": "^4.0.0",
     "command-line-args": "^6.0.1",
     "command-line-usage": "^7.0.3",
+    "csv-parser": "^3.2.0",
     "joi": "^18.0.1",
     "lz-string": "^1.5.0",
     "n-readlines": "^1.0.1",

src/cli/repl/commands/repl-commands.ts

@@ -20,6 +20,7 @@ import { guard } from '../../../util/assert';
 import { scripts } from '../../common/scripts-info';
 import { lineageCommand } from './repl-lineage';
 import { queryCommand, queryStarCommand } from './repl-query';
+import { replTurcotteTypeParseCommand } from './repl-turcotte-type-parse';
 
 function printHelpForScript(script: [string, ReplCommand], f: OutputFormatter, starredVersion?: ReplCommand): string {
 	let base = `  ${bold(padCmd(':' + script[0] + (starredVersion ? '[*]' : '')), f)}${script[1].description}`;
@@ -80,24 +81,25 @@ You can combine commands by separating them with a semicolon ${bold(';',output.f
  * All commands that should be available in the REPL.
  */
 const _commands: Record<string, ReplCommand> = {
-	'help':            helpCommand,
-	'quit':            quitCommand,
-	'version':         versionCommand,
-	'execute':         executeCommand,
-	'parse':           parseCommand,
-	'normalize':       normalizeCommand,
-	'normalize*':      normalizeStarCommand,
-	'dataflow':        dataflowCommand,
-	'dataflow*':       dataflowStarCommand,
-	'dataflowsimple':  dataflowSimplifiedCommand,
-	'dataflowsimple*': dataflowSimpleStarCommand,
-	'controlflow':     controlflowCommand,
-	'controlflow*':    controlflowStarCommand,
-	'controlflowbb':   controlflowBbCommand,
-	'controlflowbb*':  controlflowBbStarCommand,
-	'lineage':         lineageCommand,
-	'query':           queryCommand,
-	'query*':          queryStarCommand
+	'help':                helpCommand,
+	'quit':                quitCommand,
+	'version':             versionCommand,
+	'turcotte-type-parse': replTurcotteTypeParseCommand,
+	'execute':             executeCommand,
+	'parse':               parseCommand,
+	'normalize':           normalizeCommand,
+	'normalize*':          normalizeStarCommand,
+	'dataflow':            dataflowCommand,
+	'dataflow*':           dataflowStarCommand,
+	'dataflowsimple':      dataflowSimplifiedCommand,
+	'dataflowsimple*':     dataflowSimpleStarCommand,
+	'controlflow':         controlflowCommand,
+	'controlflow*':        controlflowStarCommand,
+	'controlflowbb':       controlflowBbCommand,
+	'controlflowbb*':      controlflowBbStarCommand,
+	'lineage':             lineageCommand,
+	'query':               queryCommand,
+	'query*':              queryStarCommand
 };
 let commandsInitialized = false;
 

src/cli/repl/commands/repl-turcotte-type-parse.ts

@@ -0,0 +1,69 @@
+import type { ReplCommand } from './repl-main';
+import { findSource } from '../../../dataflow/internal/process/functions/call/built-in/built-in-source';
+import fs from 'fs';
+import csvParser from 'csv-parser';
+import type { TurcotteCsvRow } from '../../../typing/adapter/turcotte-types';
+import {
+	dumpRohdeTypesFromTurcotte,
+	recoverRohdeTypesFromTurcotteFromDump,
+	turcotte2RohdeTypes
+} from '../../../typing/adapter/turcotte-types';
+import { splitAtEscapeSensitive } from '../../../util/text/args';
+import { compressToUTF16, decompressFromUTF16 } from 'lz-string';
+import type { RohdeTypes } from '../../../typing/adapter/interface';
+
+
+export const replTurcotteTypeParseCommand: ReplCommand = {
+	description:  'Give me a file to read from and I will happily print you the types in the Rohde System!',
+	usageExample: ':turcotte-type-parse foo.csv out.data',
+	aliases:      [ 'ttp',],
+	script:       false,
+	fn:           async(info) => {
+		const now = new Date();
+		if(!info.remainingLine.trim()) {
+			info.output.stderr('Please provide a file to read from. You do not need a prefix, just the file path.');
+			return;
+		}
+		const args = splitAtEscapeSensitive(info.remainingLine.trim());
+		if(args.length < 1 || args.length > 2) {
+			info.output.stderr(`Expected a single file to read from and and optional one to write to, got: ${JSON.stringify(args)}`);
+			return;
+		}
+		const [readFilePath, writeFilePath] = args as [string, string | undefined];
+		const readFile = findSource(undefined, readFilePath.trim(), { referenceChain: [] });
+		if(readFile?.length !== 1) {
+			info.output.stderr(`Could not find a single file to read from. Got: ${JSON.stringify(readFile)}`);
+			return;
+		}
+		const writeFile = writeFilePath ? findSource(undefined, writeFilePath.trim(), { referenceChain: [] }) : undefined;
+		if(writeFilePath && writeFile && writeFile.length > 0) {
+			info.output.stderr(`The output file already exists, will not overwrite: ${writeFilePath}`);
+			return;
+		}
+
+
+		const data: TurcotteCsvRow[] = [];
+		await new Promise(resolve => {
+			fs.createReadStream(readFile[0], { encoding: 'utf-8' })
+				.pipe(csvParser({ separator: ',' }))
+				.on('data', (row: TurcotteCsvRow) => {
+					data.push(row);
+				})
+				.on('end', () => resolve(null));
+		});
+		const rohdeTypes: RohdeTypes = turcotte2RohdeTypes(data);
+
+		info.output.stdout(`Parsed ${rohdeTypes.info.length} functions from ${readFile[0]} in ${Date.now() - now.getTime()}ms.`);
+
+		if(writeFilePath) {
+			const startWrite = Date.now();
+			fs.writeFileSync(writeFilePath, compressToUTF16(dumpRohdeTypesFromTurcotte(rohdeTypes)), { encoding: 'utf-16le' });
+			info.output.stdout(`Wrote ${rohdeTypes.info.length} functions to ${writeFilePath} in ${Date.now() - startWrite}ms.`);
+			const startRead = Date.now();
+			const loadCheck = decompressFromUTF16(fs.readFileSync(writeFilePath, { encoding: 'utf-16le' }));
+			const recovered = recoverRohdeTypesFromTurcotteFromDump(loadCheck);
+			info.output.stdout(`[Test] Recovered ${recovered.info.length} functions from ${writeFilePath} in ${Date.now() - startRead}ms.`);
+		}
+	}
+};
+

src/documentation/print-query-wiki.ts

@@ -36,6 +36,7 @@ import { Q } from '../search/flowr-search-builder';
 import { VertexType } from '../dataflow/graph/vertex';
 import { getTypesFromFolder, shortLink } from './doc-util/doc-types';
 import path from 'path';
+import { executeDatatypeQuery } from '../queries/catalog/datatype-query/datatype-query-executor';
 import { executeControlFlowQuery } from '../queries/catalog/control-flow-query/control-flow-query-executor';
 import { printCfgCode } from './doc-util/doc-cfg';
 import { executeDfShapeQuery } from '../queries/catalog/df-shape-query/df-shape-query-executor';
@@ -536,6 +537,30 @@ This query replaces the old [\`request-slice\`](${FlowrWikiBaseRef}/Interface#me
 	}
 });
 
+
+registerQueryDocumentation('datatype', {
+	name:             'Datatype Query',
+	type:             'active',
+	shortDescription: 'Returns all datatypes for syntactic elements (or the type for a criterion).',
+	functionName:     executeDatatypeQuery.name,
+	functionFile:     '../queries/catalog/datatype-query/datatype-query-executor.ts',
+	buildExplanation: async(shell: RShell) => {
+		const exampleCode = 'x <- 1\ny <- 2\nx';
+		return `
+This query returns the datatypes of syntactic elements in the code.
+To exemplify the capabilities, consider the following code:
+${codeBlock('r', exampleCode)}
+To see the type of the variable \`x\`, you can use the following query:
+${
+	await showQuery(shell, exampleCode, [{
+		type:     'datatype',
+		criteria: ['3@x']
+	}], { showCode: false })
+}
+`;
+	}
+});
+
 registerQueryDocumentation('dependencies', {
 	name:             'Dependencies Query',
 	type:             'active',

src/queries/catalog/datatype-query/datatype-query-executor.ts

@@ -0,0 +1,66 @@
+import type { DatatypeQuery, DatatypeQueryResult } from './datatype-query-format';
+import { log } from '../../../util/log';
+import type { BasicQueryData } from '../../base-query-format';
+import type { NormalizedAst, ParentInformation } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate';
+import type { SingleSlicingCriterion } from '../../../slicing/criterion/parse';
+import { slicingCriterionToId } from '../../../slicing/criterion/parse';
+import { inferDataTypesWithUnification } from '../../../typing/unification/infer';
+import { inferDataTypes } from '../../../typing/subtyping/infer';
+import type { UnresolvedDataType } from '../../../typing/subtyping/types';
+import { loadTracedTypes, loadTurcotteTypes } from '../../../typing/adapter/load-type-signatures';
+import fs from 'fs';
+import { superBigJsonStringify } from '../../../util/json';
+
+export async function executeDatatypeQuery({ dataflow, ast }: BasicQueryData, queries: readonly DatatypeQuery[]): Promise<DatatypeQueryResult> {
+	const start = Date.now();
+
+	const result: DatatypeQueryResult['inferredTypes'] = {};
+	for(const query of queries) {
+		const knownTypes = new Map<string, Set<UnresolvedDataType>>();
+		if(query.useTurcotteTypes ?? true) {
+			await loadTurcotteTypes(knownTypes);
+		}
+		if(query.useTracedTypes ?? true) {
+			await loadTracedTypes(knownTypes);
+		}
+
+		const typedAst = query.useSubtyping ?? true
+			? inferDataTypes(ast as NormalizedAst<ParentInformation & { typeVariable?: undefined }>, dataflow, knownTypes)
+			: inferDataTypesWithUnification(ast as NormalizedAst<ParentInformation & { typeVariable?: undefined }>, dataflow);
+		for(const criterion of query.criteria ?? typedAst.idMap.keys().map(id => `$${id}` as SingleSlicingCriterion)) {
+			if(result[criterion] !== undefined) {
+				log.warn('Duplicate criterion in datatype query:', criterion);
+				continue;
+			}
+
+			const node = criterion !== undefined ? typedAst.idMap.get(slicingCriterionToId(criterion, typedAst.idMap)) : typedAst.ast;
+			if(node === undefined) {
+				log.warn('Criterion not found in normalized AST:', criterion);
+				continue;
+			}
+
+			result[criterion] = node.info.inferredType;
+		}
+	}
+
+	const output = {
+		'.meta':       { timing: Date.now() - start },
+		inferredTypes: result
+	};
+
+	for(const filePath of queries.map(query => query.outputFile).filter(filePath => filePath !== undefined)) {
+		if(fs.existsSync(filePath)) {
+			const stream = fs.createWriteStream(filePath, { flags: 'w' });
+			superBigJsonStringify(output, '', str => stream.write(str));
+			await new Promise<void>((resolve, reject) => {
+				stream.end();
+				stream.on('error', reject);
+				stream.on('finish', resolve);
+			});
+		} else {
+			log.warn('Output file does not exist:', filePath);
+		}
+	}
+
+	return output;
+}

src/queries/catalog/datatype-query/datatype-query-format.ts

@@ -0,0 +1,49 @@
+import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
+import type { SingleSlicingCriterion } from '../../../slicing/criterion/parse';
+import type { QueryResults, SupportedQuery } from '../../query';
+import { bold } from '../../../util/text/ansi';
+import { printAsMs } from '../../../util/text/time';
+import Joi from 'joi';
+import { executeDatatypeQuery } from './datatype-query-executor';
+import type { DataType } from '../../../typing/types';
+import { prettyPrintDataType } from '../../../typing/pretty-print';
+
+/**
+ * Calculates the inferred data type for the given criterion.
+ */
+export interface DatatypeQuery extends BaseQueryFormat {
+	readonly type:              'datatype';
+	readonly criteria?:         SingleSlicingCriterion[];
+	readonly useSubtyping?:     boolean;
+	readonly useTurcotteTypes?: boolean;
+	readonly useTracedTypes?:   boolean;
+	readonly outputFile?:       string;
+}
+
+export interface DatatypeQueryResult extends BaseQueryResult {
+	/** Maps each criterion to the inferred data type, duplicates are ignored. */
+	readonly inferredTypes: Record<SingleSlicingCriterion, DataType>;
+}
+
+export const DatatypeQueryDefinition = {
+	executor:        executeDatatypeQuery,
+	asciiSummarizer: (formatter, _processed, queryResults, result) => {
+		const out = queryResults as QueryResults<'datatype'>['datatype'];
+		result.push(`Query: ${bold('datatype', formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
+		for(const [criterion, inferredType] of Object.entries(out.inferredTypes)) {
+			result.push(`   ╰ ${criterion}: {${prettyPrintDataType(inferredType)}}`);
+		}
+		return true;
+	},
+	flattenInvolvedNodes: () => {
+		return [];
+	},
+	schema: Joi.object({
+		type:             Joi.string().valid('datatype').required().description('The type of the query.'),
+		criteria:         Joi.array().items(Joi.string()).optional().description('The slicing criteria of the node to get the inferred data type for.'),
+		useSubtyping:     Joi.boolean().optional().default(true).description('Whether to use subtyping to infer the data type.'),
+		useTurcotteTypes: Joi.boolean().optional().default(true).description('Whether to use Turcotte types for inference.'),
+		useTracedTypes:   Joi.boolean().optional().default(true).description('Whether to use our manually traced types for inference.'),
+		outputFile:       Joi.string().optional().description('The output file to write the inferred types to.')
+	}).description('Datatype query used to extract the inferred data type for a node in the normalized AST')
+} as const satisfies SupportedQuery<'datatype'>;

src/queries/query.ts

@@ -42,6 +42,7 @@ import { OriginQueryDefinition } from './catalog/origin-query/origin-query-forma
 import type { LinterQuery } from './catalog/linter-query/linter-query-format';
 import { LinterQueryDefinition } from './catalog/linter-query/linter-query-format';
 import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id';
+import { DatatypeQueryDefinition, type DatatypeQuery } from './catalog/datatype-query/datatype-query-format';
 import type { ControlFlowQuery } from './catalog/control-flow-query/control-flow-query-format';
 import { ControlFlowQueryDefinition } from './catalog/control-flow-query/control-flow-query-format';
 import type { DfShapeQuery } from './catalog/df-shape-query/df-shape-query-format';
@@ -65,6 +66,7 @@ export type Query = CallContextQuery
 	| DataflowClusterQuery
 	| StaticSliceQuery
 	| LineageQuery
+	| DatatypeQuery
 	| DependenciesQuery
 	| LocationMapQuery
 	| HappensBeforeQuery
@@ -114,6 +116,7 @@ export const SupportedQueries = {
 	'dataflow-cluster': ClusterQueryDefinition,
 	'static-slice':     StaticSliceQueryDefinition,
 	'lineage':          LineageQueryDefinition,
+	'datatype':         DatatypeQueryDefinition,
 	'dependencies':     DependenciesQueryDefinition,
 	'location-map':     LocationMapQueryDefinition,
 	'search':           SearchQueryDefinition,

src/r-bridge/lang-4.x/ast/model/model.ts

@@ -31,7 +31,7 @@ export type NoInfo = object;
  * Will be used to reconstruct the source of the given element in the R-ast.
  * This will not be part of most comparisons as it is mainly of interest to the reconstruction of R code.
  */
-interface Source {
+export interface Source {
 	/**
    * The range is different from the assigned {@link Location} as it refers to the complete source range covered by the given
    * element.