From f11215ad2c3d21d7204bb7433a3e7bdd2d9a54a0 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Mon, 20 Oct 2025 11:53:09 +0200 Subject: [PATCH 01/28] Abort type-check when dependencies have parse errors. --- .../vscode/lsp/rascal/RascalLanguageServices.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java index 402e3f452..9584c3861 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.function.Function; @@ -166,12 +167,16 @@ IFunction makeParseTreeGetter(Evaluator e) { return e.getFunctionValueFactory().function(getParseTreeType, (t, u) -> { ISourceLocation resolvedLocation = Locations.toClientLocation((ISourceLocation) t[0]); try { - var tree = rascalTextDocumentService.getFile(resolvedLocation).getLastTreeWithoutErrors(); + var tree = rascalTextDocumentService.getFile(resolvedLocation).getCurrentTreeAsync(true).get(); if (tree != null) { return tree.get(); } } catch (ResponseErrorException e1) { // File is not open in the IDE + } catch (InterruptedException e1) { + // Thread was interrupted + } catch (ExecutionException e1) { + // Parse threw an exception } // Parse the source file try (var reader = URIResolverRegistry.getInstance().getCharacterReader(resolvedLocation)) { From c0f5351e1262d6007379b52ee5c05d09d2501f8e Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 28 Oct 2025 14:02:29 +0100 Subject: [PATCH 02/28] Take screenshot before IDE tests as well. --- rascal-vscode-extension/src/test/vscode-suite/ide.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index a8a801f9e..31ff8e343 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -58,7 +58,10 @@ describe('IDE', function () { await makeSureRascalModulesAreLoaded(); }); - beforeEach(async () => { + beforeEach(async function () { + if (this.test?.title) { + await ide.screenshot("IDE-" + this.test?.title); + } }); afterEach(async function () { From 1e24b8cfb0fea4d2ebb197b72fa5fccfa1fe0503 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 28 Oct 2025 15:18:30 +0100 Subject: [PATCH 03/28] Wait longer for type-checks in UI tests. --- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index 30cfb41f2..b73992274 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -313,7 +313,7 @@ export class IDEOperations { }, Delays.normal, "Could not open file") as Promise; } - async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = Delays.verySlow, tplFile = "" } = {}) { + async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = Delays.extremelySlow, tplFile = "" } = {}) { const lastLine = await editor.getNumberOfLines(); if (tplFile) { await ignoreFails(unlink(tplFile)); From d4e18ae0025084b521de9b331b23d8e6f439d74f Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Wed, 29 Oct 2025 13:34:24 +0100 Subject: [PATCH 04/28] Make screenshot on failure. --- rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts | 5 +++-- rascal-vscode-extension/src/test/vscode-suite/ide.test.ts | 5 +++-- rascal-vscode-extension/src/test/vscode-suite/repl.test.ts | 5 +++-- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 3 ++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts index 75f50499f..d87a233f9 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts @@ -47,8 +47,6 @@ parameterizedDescribe(function (errorRecovery: boolean) { this.timeout(Delays.extremelySlow * 2); - printRascalOutputOnFailure('Language Parametric Rascal'); - async function loadPico() { const repl = new RascalREPL(bench, driver); await repl.start(); @@ -82,6 +80,9 @@ parameterizedDescribe(function (errorRecovery: boolean) { bench = new Workbench(); await ignoreFails(browser.waitForWorkbench()); ide = new IDEOperations(browser); + + printRascalOutputOnFailure(ide, 'Language Parametric Rascal'); + await ide.load(); await loadPico(); picoFileBackup = await fs.readFile(TestWorkspace.picoFile); diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index 31ff8e343..fb97e44de 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -42,14 +42,15 @@ describe('IDE', function () { this.timeout(Delays.extremelySlow * 2); - printRascalOutputOnFailure('Rascal MPL'); - before(async () => { browser = VSBrowser.instance; driver = browser.driver; bench = new Workbench(); await browser.waitForWorkbench(); ide = new IDEOperations(browser); + + printRascalOutputOnFailure(ide, 'Rascal MPL'); + await ide.load(); // trigger rascal type checker to be sure for (const f of protectFiles) { diff --git a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts index e42c0ebf4..27d6f7c2a 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts @@ -37,13 +37,14 @@ describe('REPL', function () { this.timeout(2 * Delays.extremelySlow); - printRascalOutputOnFailure('Rascal MPL'); - before(async () => { browser = VSBrowser.instance; driver = browser.driver; bench = new Workbench(); ide = new IDEOperations(browser); + + printRascalOutputOnFailure(ide, 'Rascal MPL'); + await ide.load(); await ide.cleanup(); await browser.waitForWorkbench(); diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index b73992274..e2bd043fd 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -464,11 +464,12 @@ async function assureDebugLevelLoggingIsEnabled() { await prompt.confirm(); } -export function printRascalOutputOnFailure(channel: 'Language Parametric Rascal' | 'Rascal MPL') { +export function printRascalOutputOnFailure(ide: IDEOperations, channel: 'Language Parametric Rascal' | 'Rascal MPL') { const ZOOM_OUT_FACTOR = 5; afterEach("print output in case of failure", async function () { if (!this.currentTest || this.currentTest.state !== "failed") { return; } + await ide.screenshot(`failure - ${this.currentTest.fullTitle}`); try { for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { await new Workbench().executeCommand('workbench.action.zoomOut'); From a3c0e72a464fab94cdb8077d85d816a755a5fc7d Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Wed, 29 Oct 2025 14:08:26 +0100 Subject: [PATCH 05/28] Fix test title. --- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index e2bd043fd..575ad4e04 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -469,7 +469,7 @@ export function printRascalOutputOnFailure(ide: IDEOperations, channel: 'Languag const ZOOM_OUT_FACTOR = 5; afterEach("print output in case of failure", async function () { if (!this.currentTest || this.currentTest.state !== "failed") { return; } - await ide.screenshot(`failure - ${this.currentTest.fullTitle}`); + await ide.screenshot(`failure - ${this.currentTest.fullTitle()}`); try { for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { await new Workbench().executeCommand('workbench.action.zoomOut'); From 0e59912edb143b68157fbf0c44502647db335f8b Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Wed, 29 Oct 2025 17:13:41 +0100 Subject: [PATCH 06/28] Fix failure handler (again). --- .../src/test/vscode-suite/dsl.test.ts | 6 +- .../src/test/vscode-suite/ide.test.ts | 5 +- .../src/test/vscode-suite/repl.test.ts | 6 +- .../src/test/vscode-suite/utils.ts | 67 +++++++++---------- 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts index d87a233f9..de87c1df0 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts @@ -81,8 +81,6 @@ parameterizedDescribe(function (errorRecovery: boolean) { await ignoreFails(browser.waitForWorkbench()); ide = new IDEOperations(browser); - printRascalOutputOnFailure(ide, 'Language Parametric Rascal'); - await ide.load(); await loadPico(); picoFileBackup = await fs.readFile(TestWorkspace.picoFile); @@ -104,6 +102,10 @@ parameterizedDescribe(function (errorRecovery: boolean) { await fs.writeFile(TestWorkspace.picoFile, picoFileBackup); }); + afterEach("print output in case of test failure", + async function () { await printRascalOutputOnFailure(this, ide, "Language Parametric Rascal"); } + ); + it("have highlighting and parse errors", async function () { await ignoreFails(new Workbench().getEditorView().closeAllEditors()); const editor = await ide.openModule(TestWorkspace.picoFile); diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index fb97e44de..6ab114b23 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -49,8 +49,6 @@ describe('IDE', function () { await browser.waitForWorkbench(); ide = new IDEOperations(browser); - printRascalOutputOnFailure(ide, 'Rascal MPL'); - await ide.load(); // trigger rascal type checker to be sure for (const f of protectFiles) { @@ -75,6 +73,9 @@ describe('IDE', function () { } }); + afterEach("print output in case of test failure", + async function () { await printRascalOutputOnFailure(this, ide, "Rascal MPL"); } + ); async function makeSureRascalModulesAreLoaded(delay = Delays.verySlow) { try { diff --git a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts index 27d6f7c2a..d7be5d406 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts @@ -43,8 +43,6 @@ describe('REPL', function () { bench = new Workbench(); ide = new IDEOperations(browser); - printRascalOutputOnFailure(ide, 'Rascal MPL'); - await ide.load(); await ide.cleanup(); await browser.waitForWorkbench(); @@ -58,6 +56,10 @@ describe('REPL', function () { await ide.cleanup(); }); + afterEach("print output in case of test failure", + async function () { await printRascalOutputOnFailure(this, ide, "Rascal MPL"); } + ); + it("should open without a project", async () => { await new RascalREPL(bench, driver).start(); }).retries(2); diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index 575ad4e04..3cfc0b9e8 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -464,43 +464,40 @@ async function assureDebugLevelLoggingIsEnabled() { await prompt.confirm(); } -export function printRascalOutputOnFailure(ide: IDEOperations, channel: 'Language Parametric Rascal' | 'Rascal MPL') { - +export async function printRascalOutputOnFailure(context: Mocha.Context, ide: IDEOperations, channel: 'Language Parametric Rascal' | 'Rascal MPL') { const ZOOM_OUT_FACTOR = 5; - afterEach("print output in case of failure", async function () { - if (!this.currentTest || this.currentTest.state !== "failed") { return; } - await ide.screenshot(`failure - ${this.currentTest.fullTitle()}`); - try { - for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { - await new Workbench().executeCommand('workbench.action.zoomOut'); - } - const bbp = new BottomBarPanel(); - await bbp.maximize(); - console.log('**********************************************'); - console.log('***** Rascal MPL output for the failed tests: '); - let textLines: WebElement[] = []; - let tries = 0; - while (textLines.length === 0 && tries < 3) { - await showRascalOutput(bbp, channel); - textLines = await ignoreFails(bbp.findElements(By.className('view-line'))) ?? []; - tries++; - } - if (textLines.length === 0) { - console.log("We could not capture the output lines"); - } + if (!context.currentTest || context.currentTest.state !== "failed") { return; } + await ide.screenshot(`failure - ${context.currentTest.fullTitle()}`); + try { + for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { + await new Workbench().executeCommand('workbench.action.zoomOut'); + } + const bbp = new BottomBarPanel(); + await bbp.maximize(); + console.log('**********************************************'); + console.log('***** Rascal MPL output for the failed tests: '); + let textLines: WebElement[] = []; + let tries = 0; + while (textLines.length === 0 && tries < 3) { + await showRascalOutput(bbp, channel); + textLines = await ignoreFails(bbp.findElements(By.className('view-line'))) ?? []; + tries++; + } + if (textLines.length === 0) { + console.log("We could not capture the output lines"); + } - for (const l of textLines) { - console.log(await l.getText()); - } - await bbp.closePanel(); - } catch (e) { - console.log('Error capturing output: ', e); + for (const l of textLines) { + console.log(await l.getText()); } - finally { - console.log('*******End output*****************************'); - for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { - await new Workbench().executeCommand('workbench.action.zoomIn'); - } + await bbp.closePanel(); + } catch (e) { + console.log('Error capturing output: ', e); + } + finally { + console.log('*******End output*****************************'); + for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { + await new Workbench().executeCommand('workbench.action.zoomIn'); } - }); + } } From d1a8234cb4510c8c30c8674a601304666d9b5a49 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Wed, 29 Oct 2025 17:20:33 +0100 Subject: [PATCH 07/28] Take some extra screenshots around type-checks. --- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index 3cfc0b9e8..a15c50252 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -320,7 +320,11 @@ export class IDEOperations { } await editor.setTextAtLine(lastLine, await editor.getTextAtLine(lastLine) + " "); await sleep(50); + const fileName = await editor.getFileUri(); + await this.screenshot(`just before saving (for type-check) ${fileName}`); await editor.save(); + await sleep(50); + await this.screenshot(`just after saving (for type-check) ${fileName}`); if (waitForFinish) { const hasStatus = this.statusContains(checkName); await ignoreFails(this.driver.wait(hasStatus, Delays.normal, `${checkName} should have started after a save`)); From cdffe7d9c8753f1de24e87eb23a3eeb98f19c117 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Thu, 30 Oct 2025 14:23:23 +0100 Subject: [PATCH 08/28] Check modification. --- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index a15c50252..1f0f92a24 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -315,12 +315,14 @@ export class IDEOperations { async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = Delays.extremelySlow, tplFile = "" } = {}) { const lastLine = await editor.getNumberOfLines(); + const fileName = await editor.getTitle(); if (tplFile) { await ignoreFails(unlink(tplFile)); } + await this.screenshot(`just before modifying file (for type-check) ${fileName}`); await editor.setTextAtLine(lastLine, await editor.getTextAtLine(lastLine) + " "); + await this.screenshot(`just after modifying file (for type-check) ${fileName}`); await sleep(50); - const fileName = await editor.getFileUri(); await this.screenshot(`just before saving (for type-check) ${fileName}`); await editor.save(); await sleep(50); From 3b2ec68dec59078973e5cc667b24d8ce1e4e7f2b Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Thu, 30 Oct 2025 14:30:43 +0100 Subject: [PATCH 09/28] Increase the chance of hitting a failing test. --- .../src/test/vscode-suite/ide.test.ts | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index 6ab114b23..9c9ffa2df 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -119,7 +119,7 @@ describe('IDE', function () { await ide.hasErrorSquiggly(editor); }).retries(2); - it("error recovery works", async function() { + it.skip("error recovery works", async function() { const editor = await ide.openModule(TestWorkspace.mainFile); await ide.hasSyntaxHighlighting(editor); // Introduce two parse errors @@ -133,17 +133,17 @@ describe('IDE', function () { return ide.triggerTypeChecker(editor, {tplFile : tplFile, waitForFinish: waitForFinish }); } - it("save runs type checker", async function () { + it.skip("save runs type checker", async function () { const editor = await ide.openModule(TestWorkspace.mainFile); await triggerTypeChecker(editor, TestWorkspace.mainFileTpl, true); }); - it("type checker runs on dependencies", async() => { + it.skip("type checker runs on dependencies", async() => { const editor = await ide.openModule(TestWorkspace.libCallFile); await triggerTypeChecker(editor, TestWorkspace.libFileTpl, true); }); - it("go to definition works", async () => { + it.skip("go to definition works", async () => { const editor = await ide.openModule(TestWorkspace.mainFile); await triggerTypeChecker(editor, TestWorkspace.mainFileTpl, true); await editor.selectText("println"); @@ -161,20 +161,22 @@ describe('IDE', function () { }, Delays.slow, "We should jump to the right position"); }); - it("go to definition works across projects", async () => { - // due to a current bug, we have to make sure that the lib in the other project is correctly resolved - const libEditor = await ide.openModule(TestWorkspace.libFile); - await triggerTypeChecker(libEditor, "", true); - await bench.getEditorView().closeAllEditors(); - - const editor = await ide.openModule(TestWorkspace.libCallFile); - await triggerTypeChecker(editor, TestWorkspace.libCallFileTpl, true); - await editor.selectText("fib"); - await bench.executeCommand("Go to Definition"); - await waitForActiveEditor(path.basename(TestWorkspace.libFile), Delays.slow, "Lib.rsc should be opened for fib"); - }); + for (let i = 0; i < 10; i++) { + it(`go to definition works across projects (${i})`, async () => { + // due to a current bug, we have to make sure that the lib in the other project is correctly resolved + const libEditor = await ide.openModule(TestWorkspace.libFile); + await triggerTypeChecker(libEditor, "", true); + await bench.getEditorView().closeAllEditors(); + + const editor = await ide.openModule(TestWorkspace.libCallFile); + await triggerTypeChecker(editor, TestWorkspace.libCallFileTpl, true); + await editor.selectText("fib"); + await bench.executeCommand("Go to Definition"); + await waitForActiveEditor(path.basename(TestWorkspace.libFile), Delays.slow, "Lib.rsc should be opened for fib"); + }); + } - it("outline works", async () => { + it.skip("outline works", async () => { const editor = await ide.openModule(TestWorkspace.mainFile); await editor.moveCursor(1,1); const explorer = await (await bench.getActivityBar().getViewControl("Explorer"))!.openView(); @@ -187,7 +189,7 @@ describe('IDE', function () { }, Delays.normal, "Cursor should have moved to line that contains the println function"); }); - it ("rename works", async() => { + it.skip("rename works", async() => { const editor = await ide.openModule(TestWorkspace.libFile); await editor.moveCursor(7, 15); @@ -206,7 +208,7 @@ describe('IDE', function () { expect(editorText).to.contain("i -2"); }); - it("renaming files works", async() => { + it.skip("renaming files works", async() => { const newDir = path.join(TestWorkspace.libProject, "src", "main", "rascal", "lib"); await fs.mkdir(newDir, {recursive: true}); @@ -233,7 +235,7 @@ describe('IDE', function () { await fs.rm(newDir, {recursive: true, force: true}); }); - it("code actions work", async() => { + it.skip("code actions work", async() => { const editor = await ide.openModule(TestWorkspace.libCallFile); await editor.moveCursor(1,8); // in the module name @@ -246,7 +248,7 @@ describe('IDE', function () { } }); - it("editor contents used for open files", async() => { + it.skip("editor contents used for open files", async() => { const importerEditor = await ide.openModule(TestWorkspace.importerFile); const importeeEditor = await ide.openModule(TestWorkspace.importeeFile); @@ -257,7 +259,7 @@ describe('IDE', function () { await ide.hasErrorSquiggly(importerEditor); }); - it("errors in manifest detected", async() => { + it.skip("errors in manifest detected", async() => { const editor = await ide.openModule(TestWorkspace.manifest); await editor.setTextAtLine(2, "Project-Name: foobar"); await editor.save(); From 61c46d0499ddf89651d4d2ab269745ad49002468 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Thu, 30 Oct 2025 16:42:28 +0100 Subject: [PATCH 10/28] Revert "Increase the chance of hitting a failing test." This reverts commit 924f19533871ae4012d5a8f523c53c6fc83a4ead. --- .../src/test/vscode-suite/ide.test.ts | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index 9c9ffa2df..6ab114b23 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -119,7 +119,7 @@ describe('IDE', function () { await ide.hasErrorSquiggly(editor); }).retries(2); - it.skip("error recovery works", async function() { + it("error recovery works", async function() { const editor = await ide.openModule(TestWorkspace.mainFile); await ide.hasSyntaxHighlighting(editor); // Introduce two parse errors @@ -133,17 +133,17 @@ describe('IDE', function () { return ide.triggerTypeChecker(editor, {tplFile : tplFile, waitForFinish: waitForFinish }); } - it.skip("save runs type checker", async function () { + it("save runs type checker", async function () { const editor = await ide.openModule(TestWorkspace.mainFile); await triggerTypeChecker(editor, TestWorkspace.mainFileTpl, true); }); - it.skip("type checker runs on dependencies", async() => { + it("type checker runs on dependencies", async() => { const editor = await ide.openModule(TestWorkspace.libCallFile); await triggerTypeChecker(editor, TestWorkspace.libFileTpl, true); }); - it.skip("go to definition works", async () => { + it("go to definition works", async () => { const editor = await ide.openModule(TestWorkspace.mainFile); await triggerTypeChecker(editor, TestWorkspace.mainFileTpl, true); await editor.selectText("println"); @@ -161,22 +161,20 @@ describe('IDE', function () { }, Delays.slow, "We should jump to the right position"); }); - for (let i = 0; i < 10; i++) { - it(`go to definition works across projects (${i})`, async () => { - // due to a current bug, we have to make sure that the lib in the other project is correctly resolved - const libEditor = await ide.openModule(TestWorkspace.libFile); - await triggerTypeChecker(libEditor, "", true); - await bench.getEditorView().closeAllEditors(); - - const editor = await ide.openModule(TestWorkspace.libCallFile); - await triggerTypeChecker(editor, TestWorkspace.libCallFileTpl, true); - await editor.selectText("fib"); - await bench.executeCommand("Go to Definition"); - await waitForActiveEditor(path.basename(TestWorkspace.libFile), Delays.slow, "Lib.rsc should be opened for fib"); - }); - } + it("go to definition works across projects", async () => { + // due to a current bug, we have to make sure that the lib in the other project is correctly resolved + const libEditor = await ide.openModule(TestWorkspace.libFile); + await triggerTypeChecker(libEditor, "", true); + await bench.getEditorView().closeAllEditors(); + + const editor = await ide.openModule(TestWorkspace.libCallFile); + await triggerTypeChecker(editor, TestWorkspace.libCallFileTpl, true); + await editor.selectText("fib"); + await bench.executeCommand("Go to Definition"); + await waitForActiveEditor(path.basename(TestWorkspace.libFile), Delays.slow, "Lib.rsc should be opened for fib"); + }); - it.skip("outline works", async () => { + it("outline works", async () => { const editor = await ide.openModule(TestWorkspace.mainFile); await editor.moveCursor(1,1); const explorer = await (await bench.getActivityBar().getViewControl("Explorer"))!.openView(); @@ -189,7 +187,7 @@ describe('IDE', function () { }, Delays.normal, "Cursor should have moved to line that contains the println function"); }); - it.skip("rename works", async() => { + it ("rename works", async() => { const editor = await ide.openModule(TestWorkspace.libFile); await editor.moveCursor(7, 15); @@ -208,7 +206,7 @@ describe('IDE', function () { expect(editorText).to.contain("i -2"); }); - it.skip("renaming files works", async() => { + it("renaming files works", async() => { const newDir = path.join(TestWorkspace.libProject, "src", "main", "rascal", "lib"); await fs.mkdir(newDir, {recursive: true}); @@ -235,7 +233,7 @@ describe('IDE', function () { await fs.rm(newDir, {recursive: true, force: true}); }); - it.skip("code actions work", async() => { + it("code actions work", async() => { const editor = await ide.openModule(TestWorkspace.libCallFile); await editor.moveCursor(1,8); // in the module name @@ -248,7 +246,7 @@ describe('IDE', function () { } }); - it.skip("editor contents used for open files", async() => { + it("editor contents used for open files", async() => { const importerEditor = await ide.openModule(TestWorkspace.importerFile); const importeeEditor = await ide.openModule(TestWorkspace.importeeFile); @@ -259,7 +257,7 @@ describe('IDE', function () { await ide.hasErrorSquiggly(importerEditor); }); - it.skip("errors in manifest detected", async() => { + it("errors in manifest detected", async() => { const editor = await ide.openModule(TestWorkspace.manifest); await editor.setTextAtLine(2, "Project-Name: foobar"); await editor.save(); From 99d98bfadbfe14f3f600440721314efbd5bf1576 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Fri, 31 Oct 2025 10:22:46 +0100 Subject: [PATCH 11/28] More screenshots to track editor state. --- rascal-vscode-extension/src/test/vscode-suite/ide.test.ts | 1 + rascal-vscode-extension/src/test/vscode-suite/utils.ts | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index 6ab114b23..bf49d5ea4 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -164,6 +164,7 @@ describe('IDE', function () { it("go to definition works across projects", async () => { // due to a current bug, we have to make sure that the lib in the other project is correctly resolved const libEditor = await ide.openModule(TestWorkspace.libFile); + await ide.screenshot(`just after opening ${TestWorkspace.libFile}`); await triggerTypeChecker(libEditor, "", true); await bench.getEditorView().closeAllEditors(); diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index 1f0f92a24..b5f3ae5fb 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -278,8 +278,9 @@ export class IDEOperations { try { await new Workbench().executeCommand("workbench.action.revertAndCloseActiveEditor"); } catch (ex) { - this.screenshot("revert failed " + tryCount); - console.log("Revert failed, but we ignore it", ex); + const title = await new TextEditor().getTitle(); + this.screenshot(`revert of ${title} failed ` + tryCount); + console.log(`Revert of $(title) failed, but we ignore it`, ex); } try { let anyEditor = true; From 988164e8e52273b426b4c9666dd5cbe33a0ef468 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Fri, 31 Oct 2025 16:03:27 +0100 Subject: [PATCH 12/28] Shoot more screens. --- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index b5f3ae5fb..b93a33271 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -315,7 +315,9 @@ export class IDEOperations { } async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = Delays.extremelySlow, tplFile = "" } = {}) { + await this.screenshot(`just before getNumberOfLines (for type-check)`); const lastLine = await editor.getNumberOfLines(); + await this.screenshot(`just before getTitle (for type-check)`); const fileName = await editor.getTitle(); if (tplFile) { await ignoreFails(unlink(tplFile)); From 80916ef8e24aa30da556ecffe9ba52ff44db7072 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Mon, 3 Nov 2025 09:30:22 +0100 Subject: [PATCH 13/28] Fix log messages. --- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index b93a33271..7a7c97be6 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -280,7 +280,7 @@ export class IDEOperations { } catch (ex) { const title = await new TextEditor().getTitle(); this.screenshot(`revert of ${title} failed ` + tryCount); - console.log(`Revert of $(title) failed, but we ignore it`, ex); + console.log(`Revert of ${title} failed, but we ignore it`, ex); } try { let anyEditor = true; @@ -296,7 +296,7 @@ export class IDEOperations { } catch (ignored) { this.screenshot("open editor check failed " + tryCount); - console.log("Open editor dirtry check failed: ", ignored); + console.log("Open editor dirty check failed: ", ignored); return false; } From c544a28366f22215c441c7b064349d7e13fc3b78 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Mon, 3 Nov 2025 11:33:53 +0100 Subject: [PATCH 14/28] Always upload screenshots. --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3d1771c3b..86bfb9ca0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -117,7 +117,7 @@ jobs: - name: Upload Screenshots uses: actions/upload-artifact@v4 - if: failure() + if: always() with: name: screenshots-${{ matrix.os }} path: ./rascal-vscode-extension/uitests/screenshots/**/*.png From f281d743542ce882bb2043d748b68f9cb4b17697 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Mon, 20 Oct 2025 12:19:00 +0200 Subject: [PATCH 15/28] Improve exception handling. --- .../vscode/lsp/rascal/RascalLanguageServices.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java index 9584c3861..0b480bb1b 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java @@ -171,12 +171,11 @@ IFunction makeParseTreeGetter(Evaluator e) { if (tree != null) { return tree.get(); } - } catch (ResponseErrorException e1) { - // File is not open in the IDE + } catch (ResponseErrorException | ExecutionException e1) { + // File is not open in the IDE | Parse threw an exception + // In either case, fall through and try a direct parse } catch (InterruptedException e1) { - // Thread was interrupted - } catch (ExecutionException e1) { - // Parse threw an exception + Thread.currentThread().interrupt(); } // Parse the source file try (var reader = URIResolverRegistry.getInstance().getCharacterReader(resolvedLocation)) { From 90065d4a53bf49ab8cd4682820d9557a0a188993 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Thu, 23 Oct 2025 14:05:22 +0200 Subject: [PATCH 16/28] Improve error messages in case of parse errors. --- .../org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java index 0b480bb1b..0b69721bd 100644 --- a/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java +++ b/rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java @@ -167,7 +167,7 @@ IFunction makeParseTreeGetter(Evaluator e) { return e.getFunctionValueFactory().function(getParseTreeType, (t, u) -> { ISourceLocation resolvedLocation = Locations.toClientLocation((ISourceLocation) t[0]); try { - var tree = rascalTextDocumentService.getFile(resolvedLocation).getCurrentTreeAsync(true).get(); + var tree = rascalTextDocumentService.getFile(resolvedLocation).getCurrentTreeAsync(false).get(); if (tree != null) { return tree.get(); } From e5e30546c4757358f99c8d2d02e4e668efe2a1f3 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Mon, 3 Nov 2025 15:40:30 +0100 Subject: [PATCH 17/28] Fail when Lib.tpl is missing. --- rascal-vscode-extension/src/test/vscode-suite/ide.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index bf49d5ea4..cc2b295e7 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -165,7 +165,7 @@ describe('IDE', function () { // due to a current bug, we have to make sure that the lib in the other project is correctly resolved const libEditor = await ide.openModule(TestWorkspace.libFile); await ide.screenshot(`just after opening ${TestWorkspace.libFile}`); - await triggerTypeChecker(libEditor, "", true); + await triggerTypeChecker(libEditor, TestWorkspace.libFileTpl, true); await bench.getEditorView().closeAllEditors(); const editor = await ide.openModule(TestWorkspace.libCallFile); From 3bf9ca588e62cd6a4cdc0d579602968d9b4fcb61 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Mon, 3 Nov 2025 17:03:34 +0100 Subject: [PATCH 18/28] Screenshot after goto-def. --- rascal-vscode-extension/src/test/vscode-suite/ide.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index cc2b295e7..8ad3e09e3 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -172,6 +172,7 @@ describe('IDE', function () { await triggerTypeChecker(editor, TestWorkspace.libCallFileTpl, true); await editor.selectText("fib"); await bench.executeCommand("Go to Definition"); + await ide.screenshot("waiting for go-to-def"); await waitForActiveEditor(path.basename(TestWorkspace.libFile), Delays.slow, "Lib.rsc should be opened for fib"); }); From b5f6ff0585e10b6b902b8e7d1a65389d080b4a84 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 09:47:52 +0100 Subject: [PATCH 19/28] Work around getNumberOfLines bug. --- .../src/test/vscode-suite/utils.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index 7a7c97be6..60a047eb0 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -25,13 +25,12 @@ * POSSIBILITY OF SUCH DAMAGE. */ -import { assert } from "chai"; +import { assert, expect } from "chai"; import { stat, unlink } from "fs/promises"; -import path = require("path"); +import * as os from 'os'; import { env } from "process"; import { BottomBarPanel, By, CodeLens, EditorView, Key, Locator, TerminalView, TextEditor, VSBrowser, WebDriver, WebElement, WebElementCondition, Workbench, until } from "vscode-extension-tester"; -import * as os from 'os'; -import { expect } from 'chai'; +import path = require("path"); export async function sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); @@ -314,16 +313,21 @@ export class IDEOperations { }, Delays.normal, "Could not open file") as Promise; } + async appendSpace(editor: TextEditor, line = 1) { + const prompt = await new Workbench().openCommandPrompt(); + await prompt.setText(`:${line},10000`); + await prompt.confirm(); + await editor.typeText(' '); + } + async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = Delays.extremelySlow, tplFile = "" } = {}) { - await this.screenshot(`just before getNumberOfLines (for type-check)`); - const lastLine = await editor.getNumberOfLines(); await this.screenshot(`just before getTitle (for type-check)`); const fileName = await editor.getTitle(); if (tplFile) { await ignoreFails(unlink(tplFile)); } await this.screenshot(`just before modifying file (for type-check) ${fileName}`); - await editor.setTextAtLine(lastLine, await editor.getTextAtLine(lastLine) + " "); + await this.appendSpace(editor); await this.screenshot(`just after modifying file (for type-check) ${fileName}`); await sleep(50); await this.screenshot(`just before saving (for type-check) ${fileName}`); From a958de772c20586c905ddf9a82fd70b33e5db710 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 09:48:15 +0100 Subject: [PATCH 20/28] Remove workaround for cross-project imports. --- rascal-vscode-extension/src/test/vscode-suite/ide.test.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index 8ad3e09e3..3da8dab3e 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -162,12 +162,6 @@ describe('IDE', function () { }); it("go to definition works across projects", async () => { - // due to a current bug, we have to make sure that the lib in the other project is correctly resolved - const libEditor = await ide.openModule(TestWorkspace.libFile); - await ide.screenshot(`just after opening ${TestWorkspace.libFile}`); - await triggerTypeChecker(libEditor, TestWorkspace.libFileTpl, true); - await bench.getEditorView().closeAllEditors(); - const editor = await ide.openModule(TestWorkspace.libCallFile); await triggerTypeChecker(editor, TestWorkspace.libCallFileTpl, true); await editor.selectText("fib"); From ff887cfee80ca81d65e86e1f9b6511b7666f08c8 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 13:16:42 +0100 Subject: [PATCH 21/28] Revert "Wait longer for type-checks in UI tests." This reverts commit 1e24b8cfb0fea4d2ebb197b72fa5fccfa1fe0503. --- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index 60a047eb0..b16cdbf97 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -320,8 +320,7 @@ export class IDEOperations { await editor.typeText(' '); } - async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = Delays.extremelySlow, tplFile = "" } = {}) { - await this.screenshot(`just before getTitle (for type-check)`); + async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = Delays.verySlow, tplFile = "" } = {}) { const fileName = await editor.getTitle(); if (tplFile) { await ignoreFails(unlink(tplFile)); From f6b4c2f5358c3238974ab739b676ccf7b462d9a8 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 13:19:04 +0100 Subject: [PATCH 22/28] Clean up screenshots. --- rascal-vscode-extension/src/test/vscode-suite/ide.test.ts | 1 - rascal-vscode-extension/src/test/vscode-suite/utils.ts | 5 ----- 2 files changed, 6 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index 3da8dab3e..b5f4b2e43 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -166,7 +166,6 @@ describe('IDE', function () { await triggerTypeChecker(editor, TestWorkspace.libCallFileTpl, true); await editor.selectText("fib"); await bench.executeCommand("Go to Definition"); - await ide.screenshot("waiting for go-to-def"); await waitForActiveEditor(path.basename(TestWorkspace.libFile), Delays.slow, "Lib.rsc should be opened for fib"); }); diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index b16cdbf97..a04d4c989 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -321,18 +321,13 @@ export class IDEOperations { } async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = Delays.verySlow, tplFile = "" } = {}) { - const fileName = await editor.getTitle(); if (tplFile) { await ignoreFails(unlink(tplFile)); } - await this.screenshot(`just before modifying file (for type-check) ${fileName}`); await this.appendSpace(editor); - await this.screenshot(`just after modifying file (for type-check) ${fileName}`); await sleep(50); - await this.screenshot(`just before saving (for type-check) ${fileName}`); await editor.save(); await sleep(50); - await this.screenshot(`just after saving (for type-check) ${fileName}`); if (waitForFinish) { const hasStatus = this.statusContains(checkName); await ignoreFails(this.driver.wait(hasStatus, Delays.normal, `${checkName} should have started after a save`)); From 745a90faf8eb6864e05921f844125fb031f5df9b Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 13:19:46 +0100 Subject: [PATCH 23/28] Revert "Always upload screenshots." This reverts commit c544a28366f22215c441c7b064349d7e13fc3b78. --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 86bfb9ca0..3d1771c3b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -117,7 +117,7 @@ jobs: - name: Upload Screenshots uses: actions/upload-artifact@v4 - if: always() + if: failure() with: name: screenshots-${{ matrix.os }} path: ./rascal-vscode-extension/uitests/screenshots/**/*.png From 0ed2e4b9f41ffa4a73811b4d592e84fbfb16f04d Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 13:26:01 +0100 Subject: [PATCH 24/28] Clean up more unnecessary changes. --- rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts | 1 - rascal-vscode-extension/src/test/vscode-suite/ide.test.ts | 1 - rascal-vscode-extension/src/test/vscode-suite/repl.test.ts | 1 - rascal-vscode-extension/src/test/vscode-suite/utils.ts | 1 - 4 files changed, 4 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts index de87c1df0..a3ca2ccda 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts @@ -80,7 +80,6 @@ parameterizedDescribe(function (errorRecovery: boolean) { bench = new Workbench(); await ignoreFails(browser.waitForWorkbench()); ide = new IDEOperations(browser); - await ide.load(); await loadPico(); picoFileBackup = await fs.readFile(TestWorkspace.picoFile); diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index b5f4b2e43..941c049e5 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -48,7 +48,6 @@ describe('IDE', function () { bench = new Workbench(); await browser.waitForWorkbench(); ide = new IDEOperations(browser); - await ide.load(); // trigger rascal type checker to be sure for (const f of protectFiles) { diff --git a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts index d7be5d406..bb21267d4 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts @@ -42,7 +42,6 @@ describe('REPL', function () { driver = browser.driver; bench = new Workbench(); ide = new IDEOperations(browser); - await ide.load(); await ide.cleanup(); await browser.waitForWorkbench(); diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index a04d4c989..f62985f44 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -327,7 +327,6 @@ export class IDEOperations { await this.appendSpace(editor); await sleep(50); await editor.save(); - await sleep(50); if (waitForFinish) { const hasStatus = this.statusContains(checkName); await ignoreFails(this.driver.wait(hasStatus, Delays.normal, `${checkName} should have started after a save`)); From 971a621dcd025241be26f3d2ec52f0ce0cf89b5a Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 16:24:10 +0100 Subject: [PATCH 25/28] Revert "Fix failure handler (again)." This reverts commit 0e59912edb143b68157fbf0c44502647db335f8b. --- .../src/test/vscode-suite/dsl.test.ts | 7 +- .../src/test/vscode-suite/ide.test.ts | 6 +- .../src/test/vscode-suite/repl.test.ts | 7 +- .../src/test/vscode-suite/utils.ts | 67 ++++++++++--------- 4 files changed, 44 insertions(+), 43 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts index a3ca2ccda..d87a233f9 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts @@ -80,6 +80,9 @@ parameterizedDescribe(function (errorRecovery: boolean) { bench = new Workbench(); await ignoreFails(browser.waitForWorkbench()); ide = new IDEOperations(browser); + + printRascalOutputOnFailure(ide, 'Language Parametric Rascal'); + await ide.load(); await loadPico(); picoFileBackup = await fs.readFile(TestWorkspace.picoFile); @@ -101,10 +104,6 @@ parameterizedDescribe(function (errorRecovery: boolean) { await fs.writeFile(TestWorkspace.picoFile, picoFileBackup); }); - afterEach("print output in case of test failure", - async function () { await printRascalOutputOnFailure(this, ide, "Language Parametric Rascal"); } - ); - it("have highlighting and parse errors", async function () { await ignoreFails(new Workbench().getEditorView().closeAllEditors()); const editor = await ide.openModule(TestWorkspace.picoFile); diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index 941c049e5..f05e6776f 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -48,6 +48,9 @@ describe('IDE', function () { bench = new Workbench(); await browser.waitForWorkbench(); ide = new IDEOperations(browser); + + printRascalOutputOnFailure(ide, 'Rascal MPL'); + await ide.load(); // trigger rascal type checker to be sure for (const f of protectFiles) { @@ -72,9 +75,6 @@ describe('IDE', function () { } }); - afterEach("print output in case of test failure", - async function () { await printRascalOutputOnFailure(this, ide, "Rascal MPL"); } - ); async function makeSureRascalModulesAreLoaded(delay = Delays.verySlow) { try { diff --git a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts index bb21267d4..27d6f7c2a 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts @@ -42,6 +42,9 @@ describe('REPL', function () { driver = browser.driver; bench = new Workbench(); ide = new IDEOperations(browser); + + printRascalOutputOnFailure(ide, 'Rascal MPL'); + await ide.load(); await ide.cleanup(); await browser.waitForWorkbench(); @@ -55,10 +58,6 @@ describe('REPL', function () { await ide.cleanup(); }); - afterEach("print output in case of test failure", - async function () { await printRascalOutputOnFailure(this, ide, "Rascal MPL"); } - ); - it("should open without a project", async () => { await new RascalREPL(bench, driver).start(); }).retries(2); diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index f62985f44..493104da9 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -470,40 +470,43 @@ async function assureDebugLevelLoggingIsEnabled() { await prompt.confirm(); } -export async function printRascalOutputOnFailure(context: Mocha.Context, ide: IDEOperations, channel: 'Language Parametric Rascal' | 'Rascal MPL') { +export function printRascalOutputOnFailure(ide: IDEOperations, channel: 'Language Parametric Rascal' | 'Rascal MPL') { + const ZOOM_OUT_FACTOR = 5; - if (!context.currentTest || context.currentTest.state !== "failed") { return; } - await ide.screenshot(`failure - ${context.currentTest.fullTitle()}`); - try { - for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { - await new Workbench().executeCommand('workbench.action.zoomOut'); - } - const bbp = new BottomBarPanel(); - await bbp.maximize(); - console.log('**********************************************'); - console.log('***** Rascal MPL output for the failed tests: '); - let textLines: WebElement[] = []; - let tries = 0; - while (textLines.length === 0 && tries < 3) { - await showRascalOutput(bbp, channel); - textLines = await ignoreFails(bbp.findElements(By.className('view-line'))) ?? []; - tries++; - } - if (textLines.length === 0) { - console.log("We could not capture the output lines"); - } + afterEach("print output in case of failure", async function () { + if (!this.currentTest || this.currentTest.state !== "failed") { return; } + await ide.screenshot(`failure - ${this.currentTest.fullTitle()}`); + try { + for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { + await new Workbench().executeCommand('workbench.action.zoomOut'); + } + const bbp = new BottomBarPanel(); + await bbp.maximize(); + console.log('**********************************************'); + console.log('***** Rascal MPL output for the failed tests: '); + let textLines: WebElement[] = []; + let tries = 0; + while (textLines.length === 0 && tries < 3) { + await showRascalOutput(bbp, channel); + textLines = await ignoreFails(bbp.findElements(By.className('view-line'))) ?? []; + tries++; + } + if (textLines.length === 0) { + console.log("We could not capture the output lines"); + } - for (const l of textLines) { - console.log(await l.getText()); + for (const l of textLines) { + console.log(await l.getText()); + } + await bbp.closePanel(); + } catch (e) { + console.log('Error capturing output: ', e); } - await bbp.closePanel(); - } catch (e) { - console.log('Error capturing output: ', e); - } - finally { - console.log('*******End output*****************************'); - for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { - await new Workbench().executeCommand('workbench.action.zoomIn'); + finally { + console.log('*******End output*****************************'); + for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { + await new Workbench().executeCommand('workbench.action.zoomIn'); + } } - } + }); } From c63235b5236c310cf766d78986db7be1ff8be21b Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 16:24:12 +0100 Subject: [PATCH 26/28] Revert "Fix test title." This reverts commit a3c0e72a464fab94cdb8077d85d816a755a5fc7d. --- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index 493104da9..9742f2363 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -475,7 +475,7 @@ export function printRascalOutputOnFailure(ide: IDEOperations, channel: 'Languag const ZOOM_OUT_FACTOR = 5; afterEach("print output in case of failure", async function () { if (!this.currentTest || this.currentTest.state !== "failed") { return; } - await ide.screenshot(`failure - ${this.currentTest.fullTitle()}`); + await ide.screenshot(`failure - ${this.currentTest.fullTitle}`); try { for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { await new Workbench().executeCommand('workbench.action.zoomOut'); From c4638d0c35199ba06a1c9a583b73fb566aa0b2e7 Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 16:24:30 +0100 Subject: [PATCH 27/28] Revert "Make screenshot on failure." This reverts commit d4e18ae0025084b521de9b331b23d8e6f439d74f. --- rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts | 5 ++--- rascal-vscode-extension/src/test/vscode-suite/ide.test.ts | 5 ++--- rascal-vscode-extension/src/test/vscode-suite/repl.test.ts | 5 ++--- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 3 +-- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts index d87a233f9..75f50499f 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts @@ -47,6 +47,8 @@ parameterizedDescribe(function (errorRecovery: boolean) { this.timeout(Delays.extremelySlow * 2); + printRascalOutputOnFailure('Language Parametric Rascal'); + async function loadPico() { const repl = new RascalREPL(bench, driver); await repl.start(); @@ -80,9 +82,6 @@ parameterizedDescribe(function (errorRecovery: boolean) { bench = new Workbench(); await ignoreFails(browser.waitForWorkbench()); ide = new IDEOperations(browser); - - printRascalOutputOnFailure(ide, 'Language Parametric Rascal'); - await ide.load(); await loadPico(); picoFileBackup = await fs.readFile(TestWorkspace.picoFile); diff --git a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts index f05e6776f..c8a941ee9 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/ide.test.ts @@ -42,15 +42,14 @@ describe('IDE', function () { this.timeout(Delays.extremelySlow * 2); + printRascalOutputOnFailure('Rascal MPL'); + before(async () => { browser = VSBrowser.instance; driver = browser.driver; bench = new Workbench(); await browser.waitForWorkbench(); ide = new IDEOperations(browser); - - printRascalOutputOnFailure(ide, 'Rascal MPL'); - await ide.load(); // trigger rascal type checker to be sure for (const f of protectFiles) { diff --git a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts index 27d6f7c2a..e42c0ebf4 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/repl.test.ts @@ -37,14 +37,13 @@ describe('REPL', function () { this.timeout(2 * Delays.extremelySlow); + printRascalOutputOnFailure('Rascal MPL'); + before(async () => { browser = VSBrowser.instance; driver = browser.driver; bench = new Workbench(); ide = new IDEOperations(browser); - - printRascalOutputOnFailure(ide, 'Rascal MPL'); - await ide.load(); await ide.cleanup(); await browser.waitForWorkbench(); diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index 9742f2363..f7527a3c5 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -470,12 +470,11 @@ async function assureDebugLevelLoggingIsEnabled() { await prompt.confirm(); } -export function printRascalOutputOnFailure(ide: IDEOperations, channel: 'Language Parametric Rascal' | 'Rascal MPL') { +export function printRascalOutputOnFailure(channel: 'Language Parametric Rascal' | 'Rascal MPL') { const ZOOM_OUT_FACTOR = 5; afterEach("print output in case of failure", async function () { if (!this.currentTest || this.currentTest.state !== "failed") { return; } - await ide.screenshot(`failure - ${this.currentTest.fullTitle}`); try { for (let z = 0; z < ZOOM_OUT_FACTOR; z++) { await new Workbench().executeCommand('workbench.action.zoomOut'); From 9c89b3d89d0f2fc33b4e841b6f9cc66eb57dd07e Mon Sep 17 00:00:00 2001 From: Toine Hartman Date: Tue, 4 Nov 2025 16:34:59 +0100 Subject: [PATCH 28/28] Guard against failure. --- rascal-vscode-extension/src/test/vscode-suite/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rascal-vscode-extension/src/test/vscode-suite/utils.ts b/rascal-vscode-extension/src/test/vscode-suite/utils.ts index f7527a3c5..39117c249 100644 --- a/rascal-vscode-extension/src/test/vscode-suite/utils.ts +++ b/rascal-vscode-extension/src/test/vscode-suite/utils.ts @@ -277,7 +277,7 @@ export class IDEOperations { try { await new Workbench().executeCommand("workbench.action.revertAndCloseActiveEditor"); } catch (ex) { - const title = await new TextEditor().getTitle(); + const title = ignoreFails(new TextEditor().getTitle()) ?? 'unknown'; this.screenshot(`revert of ${title} failed ` + tryCount); console.log(`Revert of ${title} failed, but we ignore it`, ex); }