From e642d0df3ec7bd3eacbb5991b2ed028fa7f18456 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 14 Dec 2024 10:57:07 +0100 Subject: [PATCH 01/18] Started performance testing --- .../src/suite/performance.vscode.test.ts | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts new file mode 100644 index 0000000000..6216db0e45 --- /dev/null +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -0,0 +1,125 @@ +import { + asyncSafety, + type ActionDescriptor, + type ScopeType, + type SimpleScopeTypeType, +} from "@cursorless/common"; +import { openNewEditor, runCursorlessCommand } from "@cursorless/vscode-common"; +import assert from "assert"; +import * as vscode from "vscode"; +import { endToEndTestSetup } from "../endToEndTestSetup"; + +const obj = Object.fromEntries( + new Array(100) + .fill("") + .map((_, i) => [ + i.toString(), + Object.fromEntries( + new Array(100).fill("").map((_, i) => [i.toString(), "value"]), + ), + ]), +); +const content = JSON.stringify(obj, null, 2); +const numLines = content.split("\n").length; + +suite(`Performance: ${numLines} lines JSON`, async function () { + endToEndTestSetup(this); + + this.beforeEach(function () { + // console.log("before"); + // console.log(a.name); + console.log(this.test?.title); + console.log(this.test?.id); + }); + + this.afterAll(() => { + console.log("Done!"); + }); + + const scopeTypeTypes: Partial> = { + character: 100, + word: 100, + token: 100, + identifier: 100, + line: 100, + sentence: 100, + paragraph: 100, + // boundedParagraph: 100, + document: 100, + nonWhitespaceSequence: 100, + // boundedNonWhitespaceSequence: 300, + // string: 100, + map: 100, + // collectionKey: 300, + // value: 15000, + }; + + for (const [scopeTypeType, threshold] of Object.entries(scopeTypeTypes)) { + test( + `Select ${scopeTypeType}`, + asyncSafety(() => + selectScopeType( + { type: scopeTypeType as SimpleScopeTypeType }, + threshold, + ), + ), + ); + } + + // test( + // "Select any surrounding pair", + // asyncSafety(() => + // selectScopeType({ type: "surroundingPair", delimiter: "any" }, 300), + // ), + // ); + + test( + "Remove token", + asyncSafety(() => removeToken()), + ); +}); + +async function removeToken() { + await testPerformance(100, { + name: "remove", + target: { + type: "primitive", + modifiers: [{ type: "containingScope", scopeType: { type: "token" } }], + }, + }); +} + +async function selectScopeType(scopeType: ScopeType, threshold: number) { + await testPerformance(threshold, { + name: "setSelection", + target: { + type: "primitive", + modifiers: [{ type: "containingScope", scopeType }], + }, + }); +} + +async function testPerformance(threshold: number, action: ActionDescriptor) { + const editor = await openNewEditor(content, { languageId: "json" }); + const position = new vscode.Position(editor.document.lineCount - 3, 5); + const selection = new vscode.Selection(position, position); + editor.selections = [selection]; + editor.revealRange(selection); + + const start = performance.now(); + + await runCursorlessCommand({ + version: 7, + usePrePhraseSnapshot: false, + action, + }); + + const duration = Math.round(performance.now() - start); + + console.debug(`\t${duration} ms`); + + assert.ok( + duration < threshold, + `Duration ${duration}ms exceeds threshold ${threshold}ms`, + ); +} From cefd42843864c94183fd391994fbc8db662c1875 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 14 Dec 2024 16:25:14 +0100 Subject: [PATCH 02/18] Update --- .../src/suite/performance.vscode.test.ts | 77 +++++++++++-------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 6216db0e45..2a20a56f47 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -22,36 +22,47 @@ const obj = Object.fromEntries( const content = JSON.stringify(obj, null, 2); const numLines = content.split("\n").length; +const textBasedThreshold = 50; +const parseTreeThreshold = 300; +const surroundingPairThreshold = 20000; + suite(`Performance: ${numLines} lines JSON`, async function () { endToEndTestSetup(this); + let previousTitle = ""; + this.beforeEach(function () { - // console.log("before"); - // console.log(a.name); - console.log(this.test?.title); - console.log(this.test?.id); + const title = this.currentTest!.title; + if (title !== previousTitle) { + console.debug(` * ${title}`); + previousTitle = title; + } }); - this.afterAll(() => { - console.log("Done!"); - }); + test( + "Remove token", + asyncSafety(() => removeToken(textBasedThreshold)), + ); const scopeTypeTypes: Partial> = { - character: 100, - word: 100, - token: 100, - identifier: 100, - line: 100, - sentence: 100, - paragraph: 100, - // boundedParagraph: 100, - document: 100, - nonWhitespaceSequence: 100, - // boundedNonWhitespaceSequence: 300, - // string: 100, - map: 100, - // collectionKey: 300, - // value: 15000, + // Text based + character: textBasedThreshold, + word: textBasedThreshold, + token: textBasedThreshold, + identifier: textBasedThreshold, + line: textBasedThreshold, + sentence: textBasedThreshold, + paragraph: textBasedThreshold, + document: textBasedThreshold, + nonWhitespaceSequence: textBasedThreshold, + // Parse tree based + string: parseTreeThreshold, + map: parseTreeThreshold, + collectionKey: parseTreeThreshold, + value: parseTreeThreshold, + // Utilizes surrounding pair + boundedParagraph: surroundingPairThreshold, + boundedNonWhitespaceSequence: surroundingPairThreshold, }; for (const [scopeTypeType, threshold] of Object.entries(scopeTypeTypes)) { @@ -66,21 +77,19 @@ suite(`Performance: ${numLines} lines JSON`, async function () { ); } - // test( - // "Select any surrounding pair", - // asyncSafety(() => - // selectScopeType({ type: "surroundingPair", delimiter: "any" }, 300), - // ), - // ); - test( - "Remove token", - asyncSafety(() => removeToken()), + "Select any surrounding pair", + asyncSafety(() => + selectScopeType( + { type: "surroundingPair", delimiter: "any" }, + surroundingPairThreshold, + ), + ), ); }); -async function removeToken() { - await testPerformance(100, { +async function removeToken(threshold: number) { + await testPerformance(threshold, { name: "remove", target: { type: "primitive", @@ -116,7 +125,7 @@ async function testPerformance(threshold: number, action: ActionDescriptor) { const duration = Math.round(performance.now() - start); - console.debug(`\t${duration} ms`); + console.debug(` ${duration} ms`); assert.ok( duration < threshold, From d728c63957dc265721ebfdd4158da60f031fed89 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 14 Dec 2024 16:39:42 +0100 Subject: [PATCH 03/18] Update logging severity --- .../src/suite/performance.vscode.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 2a20a56f47..99d49f93ae 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -34,7 +34,7 @@ suite(`Performance: ${numLines} lines JSON`, async function () { this.beforeEach(function () { const title = this.currentTest!.title; if (title !== previousTitle) { - console.debug(` * ${title}`); + console.log(` * ${title}`); previousTitle = title; } }); @@ -125,7 +125,7 @@ async function testPerformance(threshold: number, action: ActionDescriptor) { const duration = Math.round(performance.now() - start); - console.debug(` ${duration} ms`); + console.log(` ${duration} ms`); assert.ok( duration < threshold, From be23c5d1d2ccb8463c697941d0ffe5e3d04b024c Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 14 Dec 2024 16:56:02 +0100 Subject: [PATCH 04/18] Updated thresholds --- .../src/suite/performance.vscode.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 99d49f93ae..70294c9ed8 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -22,8 +22,8 @@ const obj = Object.fromEntries( const content = JSON.stringify(obj, null, 2); const numLines = content.split("\n").length; -const textBasedThreshold = 50; -const parseTreeThreshold = 300; +const textBasedThreshold = 100; +const parseTreeThreshold = 500; const surroundingPairThreshold = 20000; suite(`Performance: ${numLines} lines JSON`, async function () { From bd26d735f9455c0f0018349ef7ce929e4b0977df Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 14 Dec 2024 17:20:33 +0100 Subject: [PATCH 05/18] update thresholds --- .../cursorless-vscode-e2e/src/suite/performance.vscode.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 70294c9ed8..9b43ae33e6 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -24,7 +24,7 @@ const numLines = content.split("\n").length; const textBasedThreshold = 100; const parseTreeThreshold = 500; -const surroundingPairThreshold = 20000; +const surroundingPairThreshold = 25000; suite(`Performance: ${numLines} lines JSON`, async function () { endToEndTestSetup(this); From 0bee3b6f27395c3e156d27dd87dcd2d2e089f75b Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 16 Dec 2024 13:31:44 +0100 Subject: [PATCH 06/18] Refactor --- .../src/suite/performance.vscode.test.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 9b43ae33e6..056511d286 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -9,15 +9,11 @@ import assert from "assert"; import * as vscode from "vscode"; import { endToEndTestSetup } from "../endToEndTestSetup"; +const value = Object.fromEntries( + new Array(100).fill("").map((_, i) => [i.toString(), "value"]), +); const obj = Object.fromEntries( - new Array(100) - .fill("") - .map((_, i) => [ - i.toString(), - Object.fromEntries( - new Array(100).fill("").map((_, i) => [i.toString(), "value"]), - ), - ]), + new Array(100).fill("").map((_, i) => [i.toString(), value]), ); const content = JSON.stringify(obj, null, 2); const numLines = content.split("\n").length; From 556be5667d42f1e2b178a7e5770a5dfe5028ea44 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 16 Dec 2024 13:34:10 +0100 Subject: [PATCH 07/18] More refactoring --- .../src/suite/performance.vscode.test.ts | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 056511d286..591f9d4c42 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -9,19 +9,13 @@ import assert from "assert"; import * as vscode from "vscode"; import { endToEndTestSetup } from "../endToEndTestSetup"; -const value = Object.fromEntries( - new Array(100).fill("").map((_, i) => [i.toString(), "value"]), -); -const obj = Object.fromEntries( - new Array(100).fill("").map((_, i) => [i.toString(), value]), -); -const content = JSON.stringify(obj, null, 2); -const numLines = content.split("\n").length; - const textBasedThreshold = 100; const parseTreeThreshold = 500; const surroundingPairThreshold = 25000; +const content = generateTestData(); +const numLines = content.split("\n").length; + suite(`Performance: ${numLines} lines JSON`, async function () { endToEndTestSetup(this); @@ -128,3 +122,13 @@ async function testPerformance(threshold: number, action: ActionDescriptor) { `Duration ${duration}ms exceeds threshold ${threshold}ms`, ); } + +function generateTestData() { + const value = Object.fromEntries( + new Array(100).fill("").map((_, i) => [i.toString(), "value"]), + ); + const obj = Object.fromEntries( + new Array(100).fill("").map((_, i) => [i.toString(), value]), + ); + return JSON.stringify(obj, null, 2); +} From 839de242ad92a95228c94cf63d873be912654a2f Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 16 Dec 2024 13:35:19 +0100 Subject: [PATCH 08/18] more fixes --- .../src/suite/performance.vscode.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 591f9d4c42..a0ba32f268 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -9,12 +9,8 @@ import assert from "assert"; import * as vscode from "vscode"; import { endToEndTestSetup } from "../endToEndTestSetup"; -const textBasedThreshold = 100; -const parseTreeThreshold = 500; -const surroundingPairThreshold = 25000; - -const content = generateTestData(); -const numLines = content.split("\n").length; +const testData = generateTestData(); +const numLines = testData.split("\n").length; suite(`Performance: ${numLines} lines JSON`, async function () { endToEndTestSetup(this); @@ -29,6 +25,10 @@ suite(`Performance: ${numLines} lines JSON`, async function () { } }); + const textBasedThreshold = 100; + const parseTreeThreshold = 500; + const surroundingPairThreshold = 25000; + test( "Remove token", asyncSafety(() => removeToken(textBasedThreshold)), @@ -99,7 +99,7 @@ async function selectScopeType(scopeType: ScopeType, threshold: number) { } async function testPerformance(threshold: number, action: ActionDescriptor) { - const editor = await openNewEditor(content, { languageId: "json" }); + const editor = await openNewEditor(testData, { languageId: "json" }); const position = new vscode.Position(editor.document.lineCount - 3, 5); const selection = new vscode.Selection(position, position); editor.selections = [selection]; From 82764d7047abe4b27b57f20aa0084663c6caa268 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 16 Dec 2024 13:54:32 +0100 Subject: [PATCH 09/18] More updates --- .../src/suite/performance.vscode.test.ts | 78 ++++++++++--------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index a0ba32f268..71758ffe6a 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -34,48 +34,39 @@ suite(`Performance: ${numLines} lines JSON`, async function () { asyncSafety(() => removeToken(textBasedThreshold)), ); - const scopeTypeTypes: Partial> = { + const fixtures: [SimpleScopeTypeType | ScopeType, number][] = [ // Text based - character: textBasedThreshold, - word: textBasedThreshold, - token: textBasedThreshold, - identifier: textBasedThreshold, - line: textBasedThreshold, - sentence: textBasedThreshold, - paragraph: textBasedThreshold, - document: textBasedThreshold, - nonWhitespaceSequence: textBasedThreshold, + ["character", textBasedThreshold], + ["word", textBasedThreshold], + ["token", textBasedThreshold], + ["identifier", textBasedThreshold], + ["line", textBasedThreshold], + ["sentence", textBasedThreshold], + ["paragraph", textBasedThreshold], + ["document", textBasedThreshold], + ["nonWhitespaceSequence", textBasedThreshold], // Parse tree based - string: parseTreeThreshold, - map: parseTreeThreshold, - collectionKey: parseTreeThreshold, - value: parseTreeThreshold, + ["string", parseTreeThreshold], + ["map", parseTreeThreshold], + ["collectionKey", parseTreeThreshold], + ["value", parseTreeThreshold], // Utilizes surrounding pair - boundedParagraph: surroundingPairThreshold, - boundedNonWhitespaceSequence: surroundingPairThreshold, - }; - - for (const [scopeTypeType, threshold] of Object.entries(scopeTypeTypes)) { + ["boundedParagraph", surroundingPairThreshold], + ["boundedNonWhitespaceSequence", surroundingPairThreshold], + [{ type: "surroundingPair", delimiter: "any" }, surroundingPairThreshold], + [ + { type: "surroundingPair", delimiter: "curlyBrackets" }, + surroundingPairThreshold, + ], + ]; + + for (const [scope, threshold] of fixtures) { + const [scopeType, title] = getScopeTypeAndTitle(scope); test( - `Select ${scopeTypeType}`, - asyncSafety(() => - selectScopeType( - { type: scopeTypeType as SimpleScopeTypeType }, - threshold, - ), - ), + `Select ${title}`, + asyncSafety(() => selectScopeType(scopeType, threshold)), ); } - - test( - "Select any surrounding pair", - asyncSafety(() => - selectScopeType( - { type: "surroundingPair", delimiter: "any" }, - surroundingPairThreshold, - ), - ), - ); }); async function removeToken(threshold: number) { @@ -123,7 +114,20 @@ async function testPerformance(threshold: number, action: ActionDescriptor) { ); } -function generateTestData() { +function getScopeTypeAndTitle( + scope: SimpleScopeTypeType | ScopeType, +): [ScopeType, string] { + if (typeof scope === "string") { + return [{ type: scope }, scope]; + } + switch (scope.type) { + case "surroundingPair": + return [scope, `${scope.type}.${scope.delimiter}`]; + } + throw Error(`Unexpected scope type: ${scope.type}`); +} + +function generateTestData(): string { const value = Object.fromEntries( new Array(100).fill("").map((_, i) => [i.toString(), "value"]), ); From 3ccd81db3b16d2420f14566606a1f54177c0272b Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 16 Dec 2024 13:59:20 +0100 Subject: [PATCH 10/18] Update logging --- .../src/suite/performance.vscode.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 71758ffe6a..1bff4f362a 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -20,7 +20,7 @@ suite(`Performance: ${numLines} lines JSON`, async function () { this.beforeEach(function () { const title = this.currentTest!.title; if (title !== previousTitle) { - console.log(` * ${title}`); + console.log(` ${title}`); previousTitle = title; } }); @@ -106,7 +106,7 @@ async function testPerformance(threshold: number, action: ActionDescriptor) { const duration = Math.round(performance.now() - start); - console.log(` ${duration} ms`); + console.log(` ${duration} ms`); assert.ok( duration < threshold, From 510f4addee62bd9724d2b2c10b8d7087fd7cfec0 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 16 Dec 2024 14:18:22 +0100 Subject: [PATCH 11/18] Increase threshold --- .../cursorless-vscode-e2e/src/suite/performance.vscode.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 1bff4f362a..398d3a6f95 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -27,7 +27,7 @@ suite(`Performance: ${numLines} lines JSON`, async function () { const textBasedThreshold = 100; const parseTreeThreshold = 500; - const surroundingPairThreshold = 25000; + const surroundingPairThreshold = 30000; test( "Remove token", From c85540d8dadb6618c77ddbb2f7ed6cd7814e04df Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 21 Dec 2024 11:23:32 +0100 Subject: [PATCH 12/18] Added collection item --- .../cursorless-vscode-e2e/src/suite/performance.vscode.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 398d3a6f95..45dfd7a0e5 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -53,6 +53,7 @@ suite(`Performance: ${numLines} lines JSON`, async function () { // Utilizes surrounding pair ["boundedParagraph", surroundingPairThreshold], ["boundedNonWhitespaceSequence", surroundingPairThreshold], + ["collectionItem", surroundingPairThreshold], [{ type: "surroundingPair", delimiter: "any" }, surroundingPairThreshold], [ { type: "surroundingPair", delimiter: "curlyBrackets" }, From 962dd94c2e01c8a7db7a1217b6e0b1ea94711d5e Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 21 Dec 2024 11:24:28 +0100 Subject: [PATCH 13/18] Update comments --- .../cursorless-vscode-e2e/src/suite/performance.vscode.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 45dfd7a0e5..ac15023112 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -50,10 +50,11 @@ suite(`Performance: ${numLines} lines JSON`, async function () { ["map", parseTreeThreshold], ["collectionKey", parseTreeThreshold], ["value", parseTreeThreshold], - // Utilizes surrounding pair + // Text based, but utilizes surrounding pair ["boundedParagraph", surroundingPairThreshold], ["boundedNonWhitespaceSequence", surroundingPairThreshold], ["collectionItem", surroundingPairThreshold], + // Surrounding pair [{ type: "surroundingPair", delimiter: "any" }, surroundingPairThreshold], [ { type: "surroundingPair", delimiter: "curlyBrackets" }, From 223157ad30fc43e7f71004424e62ad1ffab2b2f0 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 21 Dec 2024 18:05:53 +0100 Subject: [PATCH 14/18] Update --- .../src/suite/performance.vscode.test.ts | 52 +++++++++++-------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index ac15023112..bdb2a5a534 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -25,40 +25,40 @@ suite(`Performance: ${numLines} lines JSON`, async function () { } }); - const textBasedThreshold = 100; - const parseTreeThreshold = 500; - const surroundingPairThreshold = 30000; + const textBasedThresholdMs = 100; + const parseTreeThresholdMs = 500; + const surroundingPairThresholdMs = 30000; test( "Remove token", - asyncSafety(() => removeToken(textBasedThreshold)), + asyncSafety(() => removeToken(textBasedThresholdMs)), ); const fixtures: [SimpleScopeTypeType | ScopeType, number][] = [ // Text based - ["character", textBasedThreshold], - ["word", textBasedThreshold], - ["token", textBasedThreshold], - ["identifier", textBasedThreshold], - ["line", textBasedThreshold], - ["sentence", textBasedThreshold], - ["paragraph", textBasedThreshold], - ["document", textBasedThreshold], - ["nonWhitespaceSequence", textBasedThreshold], + ["character", textBasedThresholdMs], + ["word", textBasedThresholdMs], + ["token", textBasedThresholdMs], + ["identifier", textBasedThresholdMs], + ["line", textBasedThresholdMs], + ["sentence", textBasedThresholdMs], + ["paragraph", textBasedThresholdMs], + ["document", textBasedThresholdMs], + ["nonWhitespaceSequence", textBasedThresholdMs], // Parse tree based - ["string", parseTreeThreshold], - ["map", parseTreeThreshold], - ["collectionKey", parseTreeThreshold], - ["value", parseTreeThreshold], + ["string", parseTreeThresholdMs], + ["map", parseTreeThresholdMs], + ["collectionKey", parseTreeThresholdMs], + ["value", parseTreeThresholdMs], // Text based, but utilizes surrounding pair - ["boundedParagraph", surroundingPairThreshold], - ["boundedNonWhitespaceSequence", surroundingPairThreshold], - ["collectionItem", surroundingPairThreshold], + ["boundedParagraph", surroundingPairThresholdMs], + ["boundedNonWhitespaceSequence", surroundingPairThresholdMs], + ["collectionItem", surroundingPairThresholdMs], // Surrounding pair - [{ type: "surroundingPair", delimiter: "any" }, surroundingPairThreshold], + [{ type: "surroundingPair", delimiter: "any" }, surroundingPairThresholdMs], [ { type: "surroundingPair", delimiter: "curlyBrackets" }, - surroundingPairThreshold, + surroundingPairThresholdMs, ], ]; @@ -129,6 +129,14 @@ function getScopeTypeAndTitle( throw Error(`Unexpected scope type: ${scope.type}`); } +/** + * Generate a large JSON object with 100 keys, each with 100 values. + * { + * "0": { "0": "value", "1": "value", ... }, + * "1": { "0": "value", "1": "value", ... }, + * ... + * } + */ function generateTestData(): string { const value = Object.fromEntries( new Array(100).fill("").map((_, i) => [i.toString(), "value"]), From 637f9b6aa34dae7d7c14820c6372e26e1885a030 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sat, 21 Dec 2024 19:13:35 +0100 Subject: [PATCH 15/18] Update comment --- .../src/suite/performance.vscode.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index bdb2a5a534..c2a91c4888 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -132,9 +132,9 @@ function getScopeTypeAndTitle( /** * Generate a large JSON object with 100 keys, each with 100 values. * { - * "0": { "0": "value", "1": "value", ... }, - * "1": { "0": "value", "1": "value", ... }, + * "0": { "0": "value", ..., "99": "value" }, * ... + * * "99": { "0": "value", ..., "99": "value" }, * } */ function generateTestData(): string { From d7a82a18cf4db98b19aa02054428e664e34384a9 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 22 Dec 2024 07:14:12 +0100 Subject: [PATCH 16/18] Use shorter test data for surrounding pairs --- .../src/suite/performance.vscode.test.ts | 89 ++++++++++--------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index c2a91c4888..556cd77edd 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -9,10 +9,17 @@ import assert from "assert"; import * as vscode from "vscode"; import { endToEndTestSetup } from "../endToEndTestSetup"; -const testData = generateTestData(); -const numLines = testData.split("\n").length; +const testData = generateTestData(100); +const shortTestData = generateTestData(30); -suite(`Performance: ${numLines} lines JSON`, async function () { +const textBasedConfig: TestConfig = { thresholdMs: 100, testData }; +const parseTreeConfig: TestConfig = { thresholdMs: 500, testData }; +const surroundingPairConfig: TestConfig = { + thresholdMs: 500, + testData: shortTestData, +}; + +suite("Performance", async function () { endToEndTestSetup(this); let previousTitle = ""; @@ -25,40 +32,36 @@ suite(`Performance: ${numLines} lines JSON`, async function () { } }); - const textBasedThresholdMs = 100; - const parseTreeThresholdMs = 500; - const surroundingPairThresholdMs = 30000; - test( "Remove token", - asyncSafety(() => removeToken(textBasedThresholdMs)), + asyncSafety(() => removeToken(textBasedConfig)), ); - const fixtures: [SimpleScopeTypeType | ScopeType, number][] = [ + const fixtures: [SimpleScopeTypeType | ScopeType, TestConfig][] = [ // Text based - ["character", textBasedThresholdMs], - ["word", textBasedThresholdMs], - ["token", textBasedThresholdMs], - ["identifier", textBasedThresholdMs], - ["line", textBasedThresholdMs], - ["sentence", textBasedThresholdMs], - ["paragraph", textBasedThresholdMs], - ["document", textBasedThresholdMs], - ["nonWhitespaceSequence", textBasedThresholdMs], + ["character", textBasedConfig], + ["word", textBasedConfig], + ["token", textBasedConfig], + ["identifier", textBasedConfig], + ["line", textBasedConfig], + ["sentence", textBasedConfig], + ["paragraph", textBasedConfig], + ["document", textBasedConfig], + ["nonWhitespaceSequence", textBasedConfig], // Parse tree based - ["string", parseTreeThresholdMs], - ["map", parseTreeThresholdMs], - ["collectionKey", parseTreeThresholdMs], - ["value", parseTreeThresholdMs], + ["string", parseTreeConfig], + ["map", parseTreeConfig], + ["collectionKey", parseTreeConfig], + ["value", parseTreeConfig], // Text based, but utilizes surrounding pair - ["boundedParagraph", surroundingPairThresholdMs], - ["boundedNonWhitespaceSequence", surroundingPairThresholdMs], - ["collectionItem", surroundingPairThresholdMs], + ["boundedParagraph", surroundingPairConfig], + ["boundedNonWhitespaceSequence", surroundingPairConfig], + ["collectionItem", surroundingPairConfig], // Surrounding pair - [{ type: "surroundingPair", delimiter: "any" }, surroundingPairThresholdMs], + [{ type: "surroundingPair", delimiter: "any" }, surroundingPairConfig], [ { type: "surroundingPair", delimiter: "curlyBrackets" }, - surroundingPairThresholdMs, + surroundingPairConfig, ], ]; @@ -71,8 +74,8 @@ suite(`Performance: ${numLines} lines JSON`, async function () { } }); -async function removeToken(threshold: number) { - await testPerformance(threshold, { +async function removeToken(config: TestConfig) { + await testPerformance(config, { name: "remove", target: { type: "primitive", @@ -81,8 +84,8 @@ async function removeToken(threshold: number) { }); } -async function selectScopeType(scopeType: ScopeType, threshold: number) { - await testPerformance(threshold, { +async function selectScopeType(scopeType: ScopeType, config: TestConfig) { + await testPerformance(config, { name: "setSelection", target: { type: "primitive", @@ -91,7 +94,8 @@ async function selectScopeType(scopeType: ScopeType, threshold: number) { }); } -async function testPerformance(threshold: number, action: ActionDescriptor) { +async function testPerformance(config: TestConfig, action: ActionDescriptor) { + const { testData, thresholdMs } = config; const editor = await openNewEditor(testData, { languageId: "json" }); const position = new vscode.Position(editor.document.lineCount - 3, 5); const selection = new vscode.Selection(position, position); @@ -111,8 +115,8 @@ async function testPerformance(threshold: number, action: ActionDescriptor) { console.log(` ${duration} ms`); assert.ok( - duration < threshold, - `Duration ${duration}ms exceeds threshold ${threshold}ms`, + duration < thresholdMs, + `Duration ${duration}ms exceeds threshold ${thresholdMs}ms`, ); } @@ -130,19 +134,24 @@ function getScopeTypeAndTitle( } /** - * Generate a large JSON object with 100 keys, each with 100 values. + * Generate a large JSON object with n-keys, each with n-values. * { - * "0": { "0": "value", ..., "99": "value" }, + * "0": { "0": "value", ..., "n-1": "value" }, * ... - * * "99": { "0": "value", ..., "99": "value" }, + * "n-1": { "0": "value", ..., "n-1": "value" } * } */ -function generateTestData(): string { +function generateTestData(n: number): string { const value = Object.fromEntries( - new Array(100).fill("").map((_, i) => [i.toString(), "value"]), + new Array(n).fill("").map((_, i) => [i.toString(), "value"]), ); const obj = Object.fromEntries( - new Array(100).fill("").map((_, i) => [i.toString(), value]), + new Array(n).fill("").map((_, i) => [i.toString(), value]), ); return JSON.stringify(obj, null, 2); } + +interface TestConfig { + thresholdMs: number; + testData: string; +} From c274f4e637447181ee1605189b4dc796c1aa772d Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 23 Dec 2024 08:27:52 +0100 Subject: [PATCH 17/18] Use full test for surrounding pairs --- .../src/suite/performance.vscode.test.ts | 68 ++++++++----------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 556cd77edd..784f75867a 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -10,20 +10,18 @@ import * as vscode from "vscode"; import { endToEndTestSetup } from "../endToEndTestSetup"; const testData = generateTestData(100); -const shortTestData = generateTestData(30); -const textBasedConfig: TestConfig = { thresholdMs: 100, testData }; -const parseTreeConfig: TestConfig = { thresholdMs: 500, testData }; -const surroundingPairConfig: TestConfig = { - thresholdMs: 500, - testData: shortTestData, -}; +const textBasedThresholdMs = 100; +const parseTreeThresholdMs = 500; +const surroundingPairThresholdMs = 500; suite("Performance", async function () { endToEndTestSetup(this); let previousTitle = ""; + // Before each test, print the test title. This is done we have the test + // title before the test run time / duration. this.beforeEach(function () { const title = this.currentTest!.title; if (title !== previousTitle) { @@ -34,34 +32,34 @@ suite("Performance", async function () { test( "Remove token", - asyncSafety(() => removeToken(textBasedConfig)), + asyncSafety(() => removeToken(textBasedThresholdMs)), ); - const fixtures: [SimpleScopeTypeType | ScopeType, TestConfig][] = [ + const fixtures: [SimpleScopeTypeType | ScopeType, number][] = [ // Text based - ["character", textBasedConfig], - ["word", textBasedConfig], - ["token", textBasedConfig], - ["identifier", textBasedConfig], - ["line", textBasedConfig], - ["sentence", textBasedConfig], - ["paragraph", textBasedConfig], - ["document", textBasedConfig], - ["nonWhitespaceSequence", textBasedConfig], + ["character", textBasedThresholdMs], + ["word", textBasedThresholdMs], + ["token", textBasedThresholdMs], + ["identifier", textBasedThresholdMs], + ["line", textBasedThresholdMs], + ["sentence", textBasedThresholdMs], + ["paragraph", textBasedThresholdMs], + ["document", textBasedThresholdMs], + ["nonWhitespaceSequence", textBasedThresholdMs], // Parse tree based - ["string", parseTreeConfig], - ["map", parseTreeConfig], - ["collectionKey", parseTreeConfig], - ["value", parseTreeConfig], + ["string", parseTreeThresholdMs], + ["map", parseTreeThresholdMs], + ["collectionKey", parseTreeThresholdMs], + ["value", parseTreeThresholdMs], // Text based, but utilizes surrounding pair - ["boundedParagraph", surroundingPairConfig], - ["boundedNonWhitespaceSequence", surroundingPairConfig], - ["collectionItem", surroundingPairConfig], + ["boundedParagraph", surroundingPairThresholdMs], + ["boundedNonWhitespaceSequence", surroundingPairThresholdMs], + ["collectionItem", surroundingPairThresholdMs], // Surrounding pair - [{ type: "surroundingPair", delimiter: "any" }, surroundingPairConfig], + [{ type: "surroundingPair", delimiter: "any" }, surroundingPairThresholdMs], [ { type: "surroundingPair", delimiter: "curlyBrackets" }, - surroundingPairConfig, + surroundingPairThresholdMs, ], ]; @@ -74,8 +72,8 @@ suite("Performance", async function () { } }); -async function removeToken(config: TestConfig) { - await testPerformance(config, { +async function removeToken(thresholdMs: number) { + await testPerformance(thresholdMs, { name: "remove", target: { type: "primitive", @@ -84,8 +82,8 @@ async function removeToken(config: TestConfig) { }); } -async function selectScopeType(scopeType: ScopeType, config: TestConfig) { - await testPerformance(config, { +async function selectScopeType(scopeType: ScopeType, thresholdMs: number) { + await testPerformance(thresholdMs, { name: "setSelection", target: { type: "primitive", @@ -94,8 +92,7 @@ async function selectScopeType(scopeType: ScopeType, config: TestConfig) { }); } -async function testPerformance(config: TestConfig, action: ActionDescriptor) { - const { testData, thresholdMs } = config; +async function testPerformance(thresholdMs: number, action: ActionDescriptor) { const editor = await openNewEditor(testData, { languageId: "json" }); const position = new vscode.Position(editor.document.lineCount - 3, 5); const selection = new vscode.Selection(position, position); @@ -150,8 +147,3 @@ function generateTestData(n: number): string { ); return JSON.stringify(obj, null, 2); } - -interface TestConfig { - thresholdMs: number; - testData: string; -} From ddaf0d5ac16c66deeb86a69ccccfbc892c5f8890 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 23 Dec 2024 08:30:42 +0100 Subject: [PATCH 18/18] Updated comment --- .../cursorless-vscode-e2e/src/suite/performance.vscode.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts index 784f75867a..010a63897b 100644 --- a/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts +++ b/packages/cursorless-vscode-e2e/src/suite/performance.vscode.test.ts @@ -94,6 +94,7 @@ async function selectScopeType(scopeType: ScopeType, thresholdMs: number) { async function testPerformance(thresholdMs: number, action: ActionDescriptor) { const editor = await openNewEditor(testData, { languageId: "json" }); + // This is the position of the last json key in the document const position = new vscode.Position(editor.document.lineCount - 3, 5); const selection = new vscode.Selection(position, position); editor.selections = [selection];