From bd83147fbc8bd0c062cf7ed1184d391c8639dc5e Mon Sep 17 00:00:00 2001 From: liuyi Date: Thu, 3 Apr 2025 11:17:23 +0800 Subject: [PATCH 1/4] refactor: split getMinimumParserInfo to slice input and parser again --- src/parser/common/basicSQL.ts | 157 +++++++++++++++++++++++----------- 1 file changed, 105 insertions(+), 52 deletions(-) diff --git a/src/parser/common/basicSQL.ts b/src/parser/common/basicSQL.ts index 2cc236d9..fd9df937 100644 --- a/src/parser/common/basicSQL.ts +++ b/src/parser/common/basicSQL.ts @@ -9,6 +9,7 @@ import { ParseTreeListener, PredictionMode, ANTLRErrorListener, + Parser, } from 'antlr4ng'; import { CandidatesCollection, CodeCompletionCore } from 'antlr4-c3'; import { SQLParserBase } from '../../lib/SQLParserBase'; @@ -211,6 +212,28 @@ export abstract class BasicSQL< return this._parseTree; } + /** + * Get the parseTree of the input string. + * @param input source string + * @returns parse and parserTree + */ + private parserWithNewInput(inputSlice: string) { + const lexer = this.createLexer(inputSlice); + lexer.removeErrorListeners(); + const tokenStream = new CommonTokenStream(lexer); + tokenStream.fill(); + const parser = this.createParserFromTokenStream(tokenStream); + parser.interpreter.predictionMode = PredictionMode.SLL; + parser.removeErrorListeners(); + parser.buildParseTrees = true; + parser.errorHandler = new ErrorStrategy(); + + return { + sqlParserIns: parser, + parseTree: parser.program(), + }; + } + /** * Validate input string and return syntax errors if exists. * @param input source string @@ -263,8 +286,8 @@ export abstract class BasicSQL< if (errors.length || !this._parseTree) { return null; } - const splitListener = this.splitListener; + const splitListener = this.splitListener; this.listen(splitListener, this._parseTree); const res = splitListener.statementsContext @@ -277,35 +300,30 @@ export abstract class BasicSQL< } /** - * Get a minimum boundary parser near tokenIndex. - * @param input source string. - * @param tokenIndex start from which index to minimize the boundary. - * @param originParseTree the parse tree need to be minimized, default value is the result of parsing `input`. - * @returns minimum parser info + * Get the minimum input string that can be parsed successfully by c3. + * @param input source string + * @param caretTokenIndex tokenIndex of caretPosition + * @param originParseTree origin parseTree + * @returns MinimumInputInfo */ - public getMinimumParserInfo( + public getMinimumInputInfo( input: string, - tokenIndex: number, - originParseTree?: ParserRuleContext | null - ) { - if (arguments.length <= 2) { - this.parseWithCache(input); - originParseTree = this._parseTree; - } - + caretTokenIndex: number, + originParseTree: ParserRuleContext | undefined + ): { input: string; tokenIndexOffset: number; statementCount: number } | null { if (!originParseTree || !input?.length) return null; + let inputSlice = input; - const splitListener = this.splitListener; /** * Split sql by statement. * Try to collect candidates in as small a range as possible. */ + const splitListener = this.splitListener; this.listen(splitListener, originParseTree); + const statementCount = splitListener.statementsContext?.length; const statementsContext = splitListener.statementsContext; let tokenIndexOffset = 0; - let sqlParserIns = this._parser; - let parseTree = originParseTree; // If there are multiple statements. if (statementCount > 1) { @@ -330,14 +348,14 @@ export abstract class BasicSQL< const isNextCtxValid = index === statementCount - 1 || !statementsContext[index + 1]?.exception; - if (ctx.stop && ctx.stop.tokenIndex < tokenIndex && isPrevCtxValid) { + if (ctx.stop && ctx.stop.tokenIndex < caretTokenIndex && isPrevCtxValid) { startStatement = ctx; } if ( ctx.start && !stopStatement && - ctx.start.tokenIndex > tokenIndex && + ctx.start.tokenIndex > caretTokenIndex && isNextCtxValid ) { stopStatement = ctx; @@ -347,41 +365,67 @@ export abstract class BasicSQL< // A boundary consisting of the index of the input. const startIndex = startStatement?.start?.start ?? 0; - const stopIndex = stopStatement?.stop?.stop ?? input.length - 1; + const stopIndex = stopStatement?.stop?.stop ?? inputSlice.length - 1; /** * Save offset of the tokenIndex in the range of input * compared to the tokenIndex in the whole input */ tokenIndexOffset = startStatement?.start?.tokenIndex ?? 0; - tokenIndex = tokenIndex - tokenIndexOffset; + inputSlice = inputSlice.slice(startIndex, stopIndex); + } - /** - * Reparse the input fragment， - * and c3 will collect candidates in the newly generated parseTree. - */ - const inputSlice = input.slice(startIndex, stopIndex); + return { + input: inputSlice, + tokenIndexOffset, + statementCount, + }; + } - const lexer = this.createLexer(inputSlice); - lexer.removeErrorListeners(); - const tokenStream = new CommonTokenStream(lexer); - tokenStream.fill(); + /** + * Get a minimum boundary parser near caretTokenIndex. + * @param input source string. + * @param caretTokenIndex start from which index to minimize the boundary. + * @param originParseTree the parse tree need to be minimized, default value is the result of parsing `input`. + * @returns minimum parser info + */ + public getMinimumParserInfo( + input: string, + caretTokenIndex: number, + originParseTree: ParserRuleContext | undefined + ): { + parser: Parser; + parseTree: ParserRuleContext; + tokenIndexOffset: number; + newTokenIndex: number; + } | null { + if (!originParseTree || !input?.length) return null; - const parser = this.createParserFromTokenStream(tokenStream); - parser.interpreter.predictionMode = PredictionMode.SLL; - parser.removeErrorListeners(); - parser.buildParseTrees = true; - parser.errorHandler = new ErrorStrategy(); + const inputInfo = this.getMinimumInputInfo(input, caretTokenIndex, originParseTree); + if (!inputInfo) return null; + const { input: inputSlice, tokenIndexOffset } = inputInfo; + caretTokenIndex = caretTokenIndex - tokenIndexOffset; - sqlParserIns = parser; - parseTree = parser.program(); + let sqlParserIns = this._parser; + let parseTree = originParseTree; + + /** + * Reparse the input fragment, + * and c3 will collect candidates in the newly generated parseTree when input changed. + */ + if (inputSlice !== input) { + const { sqlParserIns: _sqlParserIns, parseTree: _parseTree } = + this.parserWithNewInput(inputSlice); + + sqlParserIns = _sqlParserIns; + parseTree = _parseTree; } return { parser: sqlParserIns, parseTree, tokenIndexOffset, - newTokenIndex: tokenIndex, + newTokenIndex: caretTokenIndex, }; } @@ -396,32 +440,41 @@ export abstract class BasicSQL< caretPosition: CaretPosition ): Suggestions | null { this.parseWithCache(input); - if (!this._parseTree) return null; - const allTokens = this.getAllTokens(input); + let allTokens = this.getAllTokens(input); let caretTokenIndex = findCaretTokenIndex(caretPosition, allTokens); - if (!caretTokenIndex && caretTokenIndex !== 0) return null; - const minimumParser = this.getMinimumParserInfo(input, caretTokenIndex); + const inputInfo = this.getMinimumInputInfo(input, caretTokenIndex, this._parseTree); + if (!inputInfo) return null; + const { input: _input, tokenIndexOffset } = inputInfo; + caretTokenIndex = caretTokenIndex - tokenIndexOffset; + let inputSlice = _input; - if (!minimumParser) return null; + let sqlParserIns = this._parser; + let parseTree = this._parseTree; + + /** + * Reparse the input fragment, + * and c3 will collect candidates in the newly generated parseTree when input changed. + */ + if (inputSlice !== input) { + const { sqlParserIns: _sqlParserIns, parseTree: _parseTree } = + this.parserWithNewInput(inputSlice); + + sqlParserIns = _sqlParserIns; + parseTree = _parseTree; + } - const { - parser: sqlParserIns, - tokenIndexOffset, - newTokenIndex, - parseTree: c3Context, - } = minimumParser; const core = new CodeCompletionCore(sqlParserIns); core.preferredRules = this.preferredRules; - const candidates = core.collectCandidates(newTokenIndex, c3Context); + const candidates = core.collectCandidates(caretTokenIndex, parseTree); const originalSuggestions = this.processCandidates( candidates, allTokens, - newTokenIndex, + caretTokenIndex, tokenIndexOffset ); From 593dbed74aa2416c2ad086f29e7e673e48fd8697 Mon Sep 17 00:00:00 2001 From: liuxy0551 Date: Fri, 27 Sep 2024 15:55:13 +0800 Subject: [PATCH 2/4] test: complete after error syntax --- .../completeAfterSyntaxError.test.ts | 65 ++++++++++++++++++ .../completeAfterSyntaxError.test.ts | 66 +++++++++++++++++++ .../completeAfterSyntaxError.test.ts | 66 +++++++++++++++++++ .../completeAfterSyntaxError.test.ts | 65 ++++++++++++++++++ .../completeAfterSyntaxError.test.ts | 65 ++++++++++++++++++ .../completeAfterSyntaxError.test.ts | 66 +++++++++++++++++++ .../completeAfterSyntaxError.test.ts | 65 ++++++++++++++++++ 7 files changed, 458 insertions(+) create mode 100644 test/parser/flink/suggestion/completeAfterSyntaxError.test.ts create mode 100644 test/parser/hive/suggestion/completeAfterSyntaxError.test.ts create mode 100644 test/parser/impala/suggestion/completeAfterSyntaxError.test.ts create mode 100644 test/parser/mysql/suggestion/completeAfterSyntaxError.test.ts create mode 100644 test/parser/postgresql/suggestion/completeAfterSyntaxError.test.ts create mode 100644 test/parser/spark/suggestion/completeAfterSyntaxError.test.ts create mode 100644 test/parser/trino/suggestion/completeAfterSyntaxError.test.ts diff --git a/test/parser/flink/suggestion/completeAfterSyntaxError.test.ts b/test/parser/flink/suggestion/completeAfterSyntaxError.test.ts new file mode 100644 index 00000000..a6def22a --- /dev/null +++ b/test/parser/flink/suggestion/completeAfterSyntaxError.test.ts @@ -0,0 +1,65 @@ +import { FlinkSQL } from 'src/parser/flink'; +import { CaretPosition, EntityContextType } from 'src/parser/common/types'; + +describe('FlinkSQL Complete After Syntax Error', () => { + const flink = new FlinkSQL(); + + const sql1 = `SELECT FROM tb2;\nINSERT INTO `; + const sql2 = `SELECT FROM tb3;\nCREATE TABLE `; + const sql3 = `SELECT FROM t1;\nSL`; + + test('Syntax error but end with semi, should suggest tableName', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 13, + }; + const suggestion = flink.getSuggestionAtCaretPosition(sql1, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords.length).toBe(0); + }); + + test('Syntax error but end with semi, should suggest tableNameCreate', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 14, + }; + const suggestion = flink.getSuggestionAtCaretPosition(sql2, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE_CREATE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords).toMatchUnorderedArray(['IF', 'IF NOT EXISTS']); + }); + + test('Syntax error but end with semi, should suggest filter token', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 2, + }; + const suggestion = flink.getSuggestionAtCaretPosition(sql3, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(0); + + // keyword + const filterKeywords = suggestion?.keywords?.filter( + (item) => item.startsWith('S') && /S(?=.*L)/.test(item) + ); + expect(filterKeywords).toMatchUnorderedArray(['SELECT']); + }); +}); diff --git a/test/parser/hive/suggestion/completeAfterSyntaxError.test.ts b/test/parser/hive/suggestion/completeAfterSyntaxError.test.ts new file mode 100644 index 00000000..93e7f9af --- /dev/null +++ b/test/parser/hive/suggestion/completeAfterSyntaxError.test.ts @@ -0,0 +1,66 @@ +import { HiveSQL } from 'src/parser/hive'; +import { CaretPosition, EntityContextType } from 'src/parser/common/types'; + +describe('HiveSQL Complete After Syntax Error', () => { + const hive = new HiveSQL(); + + const sql1 = `SELECT FROM tb2;\nINSERT INTO `; + const sql2 = `SELECT FROM tb3;\nCREATE TABLE `; + const sql3 = `SELECT FROM t1;\nSL`; + + test('Syntax error but end with semi, should suggest tableName', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 13, + }; + const suggestion = hive.getSuggestionAtCaretPosition(sql1, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords.length).toBe(1); + expect(keywords[0]).toBe('TABLE'); + }); + + test('Syntax error but end with semi, should suggest tableNameCreate', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 14, + }; + const suggestion = hive.getSuggestionAtCaretPosition(sql2, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE_CREATE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords).toMatchUnorderedArray(['IF', 'IF NOT EXISTS']); + }); + + test('Syntax error but end with semi, should suggest filter token', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 2, + }; + const suggestion = hive.getSuggestionAtCaretPosition(sql3, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(0); + + // keyword + const filterKeywords = suggestion?.keywords?.filter( + (item) => item.startsWith('S') && /S(?=.*L)/.test(item) + ); + expect(filterKeywords).toMatchUnorderedArray(['SELECT']); + }); +}); diff --git a/test/parser/impala/suggestion/completeAfterSyntaxError.test.ts b/test/parser/impala/suggestion/completeAfterSyntaxError.test.ts new file mode 100644 index 00000000..792e87c6 --- /dev/null +++ b/test/parser/impala/suggestion/completeAfterSyntaxError.test.ts @@ -0,0 +1,66 @@ +import { ImpalaSQL } from 'src/parser/impala'; +import { CaretPosition, EntityContextType } from 'src/parser/common/types'; + +describe('ImpalaSQL Complete After Syntax Error', () => { + const impala = new ImpalaSQL(); + + const sql1 = `SELECT FROM tb2;\nINSERT INTO `; + const sql2 = `SELECT FROM tb3;\nCREATE TABLE `; + const sql3 = `SELECT FROM t1;\nSL`; + + test('Syntax error but end with semi, should suggest tableName', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 13, + }; + const suggestion = impala.getSuggestionAtCaretPosition(sql1, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords.length).toBe(1); + expect(keywords[0]).toBe('TABLE'); + }); + + test('Syntax error but end with semi, should suggest tableNameCreate', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 14, + }; + const suggestion = impala.getSuggestionAtCaretPosition(sql2, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE_CREATE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords).toMatchUnorderedArray(['IF', 'IF NOT EXISTS']); + }); + + test('Syntax error but end with semi, should suggest filter token', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 2, + }; + const suggestion = impala.getSuggestionAtCaretPosition(sql3, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(0); + + // keyword + const filterKeywords = suggestion?.keywords?.filter( + (item) => item.startsWith('S') && /S(?=.*L)/.test(item) + ); + expect(filterKeywords).toMatchUnorderedArray(['SELECT']); + }); +}); diff --git a/test/parser/mysql/suggestion/completeAfterSyntaxError.test.ts b/test/parser/mysql/suggestion/completeAfterSyntaxError.test.ts new file mode 100644 index 00000000..7f5acc4a --- /dev/null +++ b/test/parser/mysql/suggestion/completeAfterSyntaxError.test.ts @@ -0,0 +1,65 @@ +import { MySQL } from 'src/parser/mysql'; +import { CaretPosition, EntityContextType } from 'src/parser/common/types'; + +describe('MySQL Complete After Syntax Error', () => { + const mysql = new MySQL(); + + const sql1 = `SELECT FROM tb2;\nINSERT INTO `; + const sql2 = `SELECT FROM tb3;\nCREATE TABLE `; + const sql3 = `SELECT FROM t1;\nSL`; + + test('Syntax error but end with semi, should suggest tableName', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 13, + }; + const suggestion = mysql.getSuggestionAtCaretPosition(sql1, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords.length).toBe(0); + }); + + test('Syntax error but end with semi, should suggest tableNameCreate', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 14, + }; + const suggestion = mysql.getSuggestionAtCaretPosition(sql2, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE_CREATE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords).toMatchUnorderedArray(['IF', 'IF NOT EXISTS']); + }); + + test('Syntax error but end with semi, should suggest filter token', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 2, + }; + const suggestion = mysql.getSuggestionAtCaretPosition(sql3, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(0); + + // keyword + const filterKeywords = suggestion?.keywords?.filter( + (item) => item.startsWith('S') && /S(?=.*L)/.test(item) + ); + expect(filterKeywords).toMatchUnorderedArray(['SELECT', 'SIGNAL']); + }); +}); diff --git a/test/parser/postgresql/suggestion/completeAfterSyntaxError.test.ts b/test/parser/postgresql/suggestion/completeAfterSyntaxError.test.ts new file mode 100644 index 00000000..07f5ddac --- /dev/null +++ b/test/parser/postgresql/suggestion/completeAfterSyntaxError.test.ts @@ -0,0 +1,65 @@ +import { PostgreSQL } from 'src/parser/postgresql'; +import { CaretPosition, EntityContextType } from 'src/parser/common/types'; + +describe('PostgreSQL Complete After Syntax Error', () => { + const postgresql = new PostgreSQL(); + + const sql1 = `SELECT FROM tb2;\nINSERT INTO `; + const sql2 = `SELECT FROM tb3;\nCREATE TABLE `; + const sql3 = `SELECT FROM t1;\nSL`; + + test('Syntax error but end with semi, should suggest tableName', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 13, + }; + const suggestion = postgresql.getSuggestionAtCaretPosition(sql1, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords.length).toBe(0); + }); + + test('Syntax error but end with semi, should suggest tableNameCreate', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 14, + }; + const suggestion = postgresql.getSuggestionAtCaretPosition(sql2, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE_CREATE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords).toMatchUnorderedArray(['IF', 'IF NOT EXISTS']); + }); + + test('Syntax error but end with semi, should suggest filter token', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 2, + }; + const suggestion = postgresql.getSuggestionAtCaretPosition(sql3, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(0); + + // keyword + const filterKeywords = suggestion?.keywords?.filter( + (item) => item.startsWith('SEL') && /S(?=.*L)/.test(item) + ); + expect(filterKeywords).toMatchUnorderedArray(['SELECT']); + }); +}); diff --git a/test/parser/spark/suggestion/completeAfterSyntaxError.test.ts b/test/parser/spark/suggestion/completeAfterSyntaxError.test.ts new file mode 100644 index 00000000..fde5b27d --- /dev/null +++ b/test/parser/spark/suggestion/completeAfterSyntaxError.test.ts @@ -0,0 +1,66 @@ +import { SparkSQL } from 'src/parser/spark'; +import { CaretPosition, EntityContextType } from 'src/parser/common/types'; + +describe('SparkSQL Complete After Syntax Error', () => { + const spark = new SparkSQL(); + + const sql1 = `SELECT FROM tb2;\nINSERT INTO `; + const sql2 = `SELECT FROM tb3;\nCREATE TABLE `; + const sql3 = `SELECT FROM t1;\nSL`; + + test('Syntax error but end with semi, should suggest tableName', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 13, + }; + const suggestion = spark.getSuggestionAtCaretPosition(sql1, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords.length).toBe(1); + expect(keywords[0]).toBe('TABLE'); + }); + + test('Syntax error but end with semi, should suggest tableNameCreate', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 14, + }; + const suggestion = spark.getSuggestionAtCaretPosition(sql2, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE_CREATE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords).toMatchUnorderedArray(['IF', 'IF NOT EXISTS']); + }); + + test('Syntax error but end with semi, should suggest filter token', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 2, + }; + const suggestion = spark.getSuggestionAtCaretPosition(sql3, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(0); + + // keyword + const filterKeywords = suggestion?.keywords?.filter( + (item) => item.startsWith('S') && /S(?=.*L)/.test(item) + ); + expect(filterKeywords).toMatchUnorderedArray(['SELECT']); + }); +}); diff --git a/test/parser/trino/suggestion/completeAfterSyntaxError.test.ts b/test/parser/trino/suggestion/completeAfterSyntaxError.test.ts new file mode 100644 index 00000000..59f33213 --- /dev/null +++ b/test/parser/trino/suggestion/completeAfterSyntaxError.test.ts @@ -0,0 +1,65 @@ +import { TrinoSQL } from 'src/parser/trino'; +import { CaretPosition, EntityContextType } from 'src/parser/common/types'; + +describe('TrinoSQL Complete After Syntax Error', () => { + const trino = new TrinoSQL(); + + const sql1 = `SELECT FROM tb2;\nINSERT INTO `; + const sql2 = `SELECT FROM tb3;\nCREATE TABLE `; + const sql3 = `SELECT FROM t1;\nSL`; + + test('Syntax error but end with semi, should suggest tableName', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 13, + }; + const suggestion = trino.getSuggestionAtCaretPosition(sql1, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords.length).toBe(0); + }); + + test('Syntax error but end with semi, should suggest tableNameCreate', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 14, + }; + const suggestion = trino.getSuggestionAtCaretPosition(sql2, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(1); + expect(syntaxes[0].syntaxContextType).toBe(EntityContextType.TABLE_CREATE); + + // keyword + const keywords = suggestion?.keywords; + expect(keywords).toMatchUnorderedArray(['IF', 'IF NOT EXISTS']); + }); + + test('Syntax error but end with semi, should suggest filter token', () => { + const pos: CaretPosition = { + lineNumber: 2, + column: 2, + }; + const suggestion = trino.getSuggestionAtCaretPosition(sql3, pos); + expect(suggestion).not.toBeUndefined(); + + // syntax + const syntaxes = suggestion?.syntax; + expect(syntaxes.length).toBe(0); + + // keyword + const filterKeywords = suggestion?.keywords?.filter( + (item) => item.startsWith('S') && /S(?=.*L)/.test(item) + ); + expect(filterKeywords).toMatchUnorderedArray(['SELECT']); + }); +}); From 391573949f827400f78e213977516ffe4f11b8e4 Mon Sep 17 00:00:00 2001 From: liuyi Date: Thu, 3 Apr 2025 11:31:23 +0800 Subject: [PATCH 3/4] feat: complete after error syntax --- src/parser/common/basicSQL.ts | 105 +++++++++++++++++- src/parser/common/semanticContextCollector.ts | 3 +- 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/src/parser/common/basicSQL.ts b/src/parser/common/basicSQL.ts index fd9df937..70f48d64 100644 --- a/src/parser/common/basicSQL.ts +++ b/src/parser/common/basicSQL.ts @@ -29,6 +29,8 @@ import type { EntityCollector } from './entityCollector'; import { EntityContext } from './entityCollector'; import SemanticContextCollector from './semanticContextCollector'; +export const SQL_SPLIT_SYMBOL_TEXT = ';'; + /** * Basic SQL class, every sql needs extends it. */ @@ -286,7 +288,6 @@ export abstract class BasicSQL< if (errors.length || !this._parseTree) { return null; } - const splitListener = this.splitListener; this.listen(splitListener, this._parseTree); @@ -299,6 +300,78 @@ export abstract class BasicSQL< return res; } + /** + * Get the smaller range of input + * @param input string + * @param allTokens all tokens from input + * @param tokenIndexOffset offset of the tokenIndex in the range of input + * @param caretTokenIndex tokenIndex of caretPosition + * @returns inputSlice: string, caretTokenIndex: number + */ + private splitInputBySymbolText( + input: string, + allTokens: Token[], + tokenIndexOffset: number, + caretTokenIndex: number + ): { inputSlice: string; allTokens: Token[]; caretTokenIndex: number } { + const tokens = allTokens.slice(tokenIndexOffset); + /** + * Set startToken + */ + let startToken: Token | null = null; + for (let tokenIndex = caretTokenIndex - tokenIndexOffset; tokenIndex >= 0; tokenIndex--) { + const token = tokens[tokenIndex]; + if (token?.text === SQL_SPLIT_SYMBOL_TEXT) { + startToken = tokens[tokenIndex + 1]; + break; + } + } + if (startToken === null) { + startToken = tokens[0]; + } + + /** + * Set stopToken + */ + let stopToken: Token | null = null; + for ( + let tokenIndex = caretTokenIndex - tokenIndexOffset; + tokenIndex < tokens.length; + tokenIndex++ + ) { + const token = tokens[tokenIndex]; + if (token?.text === SQL_SPLIT_SYMBOL_TEXT) { + stopToken = token; + break; + } + } + if (stopToken === null) { + stopToken = tokens[tokens.length - 1]; + } + + const indexOffset = tokens[0].start; + let startIndex = startToken.start - indexOffset; + let stopIndex = stopToken.stop + 1 - indexOffset; + + /** + * Save offset of the tokenIndex in the range of input + * compared to the tokenIndex in the whole input + */ + const _tokenIndexOffset = startToken.tokenIndex; + const _caretTokenIndex = caretTokenIndex - _tokenIndexOffset; + + /** + * Get the smaller range of _input + */ + const _input = input.slice(startIndex, stopIndex); + + return { + inputSlice: _input, + allTokens: allTokens.slice(_tokenIndexOffset), + caretTokenIndex: _caretTokenIndex, + }; + } + /** * Get the minimum input string that can be parsed successfully by c3. * @param input source string @@ -448,10 +521,33 @@ export abstract class BasicSQL< const inputInfo = this.getMinimumInputInfo(input, caretTokenIndex, this._parseTree); if (!inputInfo) return null; - const { input: _input, tokenIndexOffset } = inputInfo; - caretTokenIndex = caretTokenIndex - tokenIndexOffset; + const { input: _input, tokenIndexOffset, statementCount } = inputInfo; let inputSlice = _input; + /** + * Split the inputSlice by separator to get the smaller range of inputSlice. + */ + if (inputSlice.includes(SQL_SPLIT_SYMBOL_TEXT)) { + const { + inputSlice: _inputSlice, + allTokens: _allTokens, + caretTokenIndex: _caretTokenIndex, + } = this.splitInputBySymbolText( + inputSlice, + allTokens, + tokenIndexOffset, + caretTokenIndex + ); + + allTokens = _allTokens; + caretTokenIndex = _caretTokenIndex; + inputSlice = _inputSlice; + } else { + if (statementCount > 1) { + caretTokenIndex = caretTokenIndex - tokenIndexOffset; + } + } + let sqlParserIns = this._parser; let parseTree = this._parseTree; @@ -475,7 +571,8 @@ export abstract class BasicSQL< candidates, allTokens, caretTokenIndex, - tokenIndexOffset + 0 + // tokenIndexOffset ); const syntaxSuggestions: SyntaxSuggestion[] = originalSuggestions.syntax.map( diff --git a/src/parser/common/semanticContextCollector.ts b/src/parser/common/semanticContextCollector.ts index 23e109bd..07b329b7 100644 --- a/src/parser/common/semanticContextCollector.ts +++ b/src/parser/common/semanticContextCollector.ts @@ -6,8 +6,7 @@ import { SemanticContext, SqlSplitStrategy, } from '../common/types'; - -export const SQL_SPLIT_SYMBOL_TEXT = ';'; +import { SQL_SPLIT_SYMBOL_TEXT } from './basicSQL'; abstract class SemanticContextCollector { constructor( From 46a51ecd54202d3378d44da2dbbeede70a5f207c Mon Sep 17 00:00:00 2001 From: liuyi Date: Mon, 7 Apr 2025 14:10:59 +0800 Subject: [PATCH 4/4] feat: use createParser to get parserIns and remove parserWithNewInput --- src/parser/common/basicSQL.ts | 36 ++++------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/src/parser/common/basicSQL.ts b/src/parser/common/basicSQL.ts index 70f48d64..686df544 100644 --- a/src/parser/common/basicSQL.ts +++ b/src/parser/common/basicSQL.ts @@ -214,28 +214,6 @@ export abstract class BasicSQL< return this._parseTree; } - /** - * Get the parseTree of the input string. - * @param input source string - * @returns parse and parserTree - */ - private parserWithNewInput(inputSlice: string) { - const lexer = this.createLexer(inputSlice); - lexer.removeErrorListeners(); - const tokenStream = new CommonTokenStream(lexer); - tokenStream.fill(); - const parser = this.createParserFromTokenStream(tokenStream); - parser.interpreter.predictionMode = PredictionMode.SLL; - parser.removeErrorListeners(); - parser.buildParseTrees = true; - parser.errorHandler = new ErrorStrategy(); - - return { - sqlParserIns: parser, - parseTree: parser.program(), - }; - } - /** * Validate input string and return syntax errors if exists. * @param input source string @@ -487,11 +465,8 @@ export abstract class BasicSQL< * and c3 will collect candidates in the newly generated parseTree when input changed. */ if (inputSlice !== input) { - const { sqlParserIns: _sqlParserIns, parseTree: _parseTree } = - this.parserWithNewInput(inputSlice); - - sqlParserIns = _sqlParserIns; - parseTree = _parseTree; + sqlParserIns = this.createParser(inputSlice); + parseTree = sqlParserIns.program(); } return { @@ -556,11 +531,8 @@ export abstract class BasicSQL< * and c3 will collect candidates in the newly generated parseTree when input changed. */ if (inputSlice !== input) { - const { sqlParserIns: _sqlParserIns, parseTree: _parseTree } = - this.parserWithNewInput(inputSlice); - - sqlParserIns = _sqlParserIns; - parseTree = _parseTree; + sqlParserIns = this.createParser(inputSlice); + parseTree = sqlParserIns.program(); } const core = new CodeCompletionCore(sqlParserIns);