Fix flaky UI tests caused by unexpected edits during type-check

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalLanguageServices.java

@@ -38,6 +38,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.function.Function;
@@ -166,12 +167,15 @@ IFunction makeParseTreeGetter(Evaluator e) {
         return e.getFunctionValueFactory().function(getParseTreeType, (t, u) -> {
             ISourceLocation resolvedLocation = Locations.toClientLocation((ISourceLocation) t[0]);
             try {
-                var tree = rascalTextDocumentService.getFile(resolvedLocation).getLastTreeWithoutErrors();
+                var tree = rascalTextDocumentService.getFile(resolvedLocation).getCurrentTreeAsync(false).get();
                 if (tree != null) {
                     return tree.get();
                 }
-            } catch (ResponseErrorException e1) {
-                // File is not open in the IDE
+            } catch (ResponseErrorException | ExecutionException e1) {
+                // File is not open in the IDE | Parse threw an exception
+                // In either case, fall through and try a direct parse
+            } catch (InterruptedException e1) {
+                Thread.currentThread().interrupt();
             }
             // Parse the source file
             try (var reader = URIResolverRegistry.getInstance().getCharacterReader(resolvedLocation)) {

rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts

@@ -47,8 +47,6 @@ parameterizedDescribe(function (errorRecovery: boolean) {
 
     this.timeout(Delays.extremelySlow * 2);
 
-    printRascalOutputOnFailure('Language Parametric Rascal');
-
     async function loadPico() {
         const repl = new RascalREPL(bench, driver);
         await repl.start();
@@ -103,6 +101,10 @@ parameterizedDescribe(function (errorRecovery: boolean) {
         await fs.writeFile(TestWorkspace.picoFile, picoFileBackup);
     });
 
+    afterEach("print output in case of test failure",
+        async function () { await printRascalOutputOnFailure(this, ide, "Language Parametric Rascal"); }
+    );
+
     it("have highlighting and parse errors", async function () {
         await ignoreFails(new Workbench().getEditorView().closeAllEditors());
         const editor = await ide.openModule(TestWorkspace.picoFile);

rascal-vscode-extension/src/test/vscode-suite/ide.test.ts

@@ -42,8 +42,6 @@ describe('IDE', function () {
 
     this.timeout(Delays.extremelySlow * 2);
 
-    printRascalOutputOnFailure('Rascal MPL');
-
     before(async () => {
         browser = VSBrowser.instance;
         driver = browser.driver;
@@ -58,7 +56,10 @@ describe('IDE', function () {
         await makeSureRascalModulesAreLoaded();
     });
 
-    beforeEach(async () => {
+    beforeEach(async function () {
+        if (this.test?.title) {
+            await ide.screenshot("IDE-" + this.test?.title);
+        }
     });
 
     afterEach(async function () {
@@ -71,6 +72,9 @@ describe('IDE', function () {
         }
     });
 
+    afterEach("print output in case of test failure",
+        async function () { await printRascalOutputOnFailure(this, ide, "Rascal MPL"); }
+    );
 
     async function makeSureRascalModulesAreLoaded(delay = Delays.verySlow) {
         try {
@@ -157,11 +161,6 @@ describe('IDE', function () {
     });
 
     it("go to definition works across projects", async () => {
-        // due to a current bug, we have to make sure that the lib in the other project is correctly resolved
-        const libEditor = await ide.openModule(TestWorkspace.libFile);
-        await triggerTypeChecker(libEditor, "", true);
-        await bench.getEditorView().closeAllEditors();
-
         const editor = await ide.openModule(TestWorkspace.libCallFile);
         await triggerTypeChecker(editor, TestWorkspace.libCallFileTpl, true);
         await editor.selectText("fib");

rascal-vscode-extension/src/test/vscode-suite/repl.test.ts

@@ -37,8 +37,6 @@ describe('REPL', function () {
 
     this.timeout(2 * Delays.extremelySlow);
 
-    printRascalOutputOnFailure('Rascal MPL');
-
     before(async () => {
         browser = VSBrowser.instance;
         driver = browser.driver;
@@ -57,6 +55,10 @@ describe('REPL', function () {
         await ide.cleanup();
     });
 
+    afterEach("print output in case of test failure",
+        async function () { await printRascalOutputOnFailure(this, ide, "Rascal MPL"); }
+    );
+
     it("should open without a project", async () => {
         await new RascalREPL(bench, driver).start();
     }).retries(2);

rascal-vscode-extension/src/test/vscode-suite/utils.ts

@@ -25,13 +25,12 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { assert } from "chai";
+import { assert, expect } from "chai";
 import { stat, unlink } from "fs/promises";
-import path = require("path");
+import * as os from 'os';
 import { env } from "process";
 import { BottomBarPanel, By, CodeLens, EditorView, Key, Locator, TerminalView, TextEditor, VSBrowser, WebDriver, WebElement, WebElementCondition, Workbench, until } from "vscode-extension-tester";
-import * as os from 'os';
-import { expect } from 'chai';
+import path = require("path");
 
 export async function sleep(ms: number) {
     return new Promise(r => setTimeout(r, ms));
@@ -278,8 +277,9 @@ export class IDEOperations {
             try {
                 await new Workbench().executeCommand("workbench.action.revertAndCloseActiveEditor");
             } catch (ex) {
-                this.screenshot("revert failed " + tryCount);
-                console.log("Revert failed, but we ignore it", ex);
+                const title = await new TextEditor().getTitle();
+                this.screenshot(`revert of ${title} failed ` + tryCount);
+                console.log(`Revert of ${title} failed, but we ignore it`, ex);
             }
             try {
                 let anyEditor = true;
@@ -295,7 +295,7 @@ export class IDEOperations {
             }
             catch (ignored) {
                 this.screenshot("open editor check failed " + tryCount);
-                console.log("Open editor dirtry check failed: ", ignored);
+                console.log("Open editor dirty check failed: ", ignored);
                 return false;
 
             }
@@ -313,12 +313,18 @@ export class IDEOperations {
         }, Delays.normal, "Could not open file") as Promise<TextEditor>;
     }
 
+    async appendSpace(editor: TextEditor, line = 1) {
+        const prompt = await new Workbench().openCommandPrompt();
+        await prompt.setText(`:${line},10000`);
+        await prompt.confirm();
+        await editor.typeText(' ');
+    }
+
     async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = Delays.verySlow, tplFile = "" } = {}) {
-        const lastLine = await editor.getNumberOfLines();
         if (tplFile) {
             await ignoreFails(unlink(tplFile));
         }
-        await editor.setTextAtLine(lastLine, await editor.getTextAtLine(lastLine) + " ");
+        await this.appendSpace(editor);
         await sleep(50);
         await editor.save();
         if (waitForFinish) {
@@ -464,42 +470,40 @@ async function assureDebugLevelLoggingIsEnabled() {
     await prompt.confirm();
 }
 
-export function printRascalOutputOnFailure(channel: 'Language Parametric Rascal' | 'Rascal MPL') {
-
+export async function printRascalOutputOnFailure(context: Mocha.Context, ide: IDEOperations, channel: 'Language Parametric Rascal' | 'Rascal MPL') {
     const ZOOM_OUT_FACTOR = 5;
-    afterEach("print output in case of failure", async function () {
-        if (!this.currentTest || this.currentTest.state !== "failed") { return; }
-        try {
-            for (let z = 0; z < ZOOM_OUT_FACTOR; z++) {
-                await new Workbench().executeCommand('workbench.action.zoomOut');
-            }
-            const bbp = new BottomBarPanel();
-            await bbp.maximize();
-            console.log('**********************************************');
-            console.log('***** Rascal MPL output for the failed tests: ');
-            let textLines: WebElement[] = [];
-            let tries = 0;
-            while (textLines.length === 0 && tries < 3) {
-                await showRascalOutput(bbp, channel);
-                textLines = await ignoreFails(bbp.findElements(By.className('view-line'))) ?? [];
-                tries++;
-            }
-            if (textLines.length === 0) {
-                console.log("We could not capture the output lines");
-            }
+    if (!context.currentTest || context.currentTest.state !== "failed") { return; }
+    await ide.screenshot(`failure - ${context.currentTest.fullTitle()}`);
+    try {
+        for (let z = 0; z < ZOOM_OUT_FACTOR; z++) {
+            await new Workbench().executeCommand('workbench.action.zoomOut');
+        }
+        const bbp = new BottomBarPanel();
+        await bbp.maximize();
+        console.log('**********************************************');
+        console.log('***** Rascal MPL output for the failed tests: ');
+        let textLines: WebElement[] = [];
+        let tries = 0;
+        while (textLines.length === 0 && tries < 3) {
+            await showRascalOutput(bbp, channel);
+            textLines = await ignoreFails(bbp.findElements(By.className('view-line'))) ?? [];
+            tries++;
+        }
+        if (textLines.length === 0) {
+            console.log("We could not capture the output lines");
+        }
 
-            for (const l of textLines) {
-                console.log(await l.getText());
-            }
-            await bbp.closePanel();
-        } catch (e) {
-            console.log('Error capturing output: ', e);
+        for (const l of textLines) {
+            console.log(await l.getText());
         }
-        finally {
-            console.log('*******End output*****************************');
-            for (let z = 0; z < ZOOM_OUT_FACTOR; z++) {
-                await new Workbench().executeCommand('workbench.action.zoomIn');
-            }
+        await bbp.closePanel();
+    } catch (e) {
+        console.log('Error capturing output: ', e);
+    }
+    finally {
+        console.log('*******End output*****************************');
+        for (let z = 0; z < ZOOM_OUT_FACTOR; z++) {
+            await new Workbench().executeCommand('workbench.action.zoomIn');
         }
-    });
+    }
 }