From dc2084276ac177eebdac985ca99433fd5998c046 Mon Sep 17 00:00:00 2001 From: mkslanc Date: Tue, 20 Aug 2024 12:49:18 +0400 Subject: [PATCH 1/6] scopes tokenizer: new implementation --- ace-internal.d.ts | 23 +++- demo/kitchen-sink/token_tooltip.js | 32 +++-- src/background_tokenizer.js | 4 +- src/edit_session/folding.js | 2 +- src/layer/text_util.js | 2 +- src/mode/folding/mixed.js | 2 +- src/mode/lua_highlight_rules.js | 52 ++++---- src/mode/text.js | 15 ++- src/mode/text_highlight_rules.js | 10 +- src/scope.js | 94 ++++++++++++++ src/tokenizer.js | 194 +++++++++++++++++++---------- src/tooltip.js | 4 +- 12 files changed, 323 insertions(+), 111 deletions(-) create mode 100644 src/scope.js diff --git a/ace-internal.d.ts b/ace-internal.d.ts index 154614fd961..8769395dcba 100644 --- a/ace-internal.d.ts +++ b/ace-internal.d.ts @@ -1217,7 +1217,7 @@ export namespace Ace { trim?: boolean, firstLineNumber?: number, showGutter?: boolean - } +} } @@ -1554,6 +1554,27 @@ declare module "./src/mouse/default_gutter_handler" { } } +declare module "./src/scope" { + + export interface Scope extends String { + name: string; + children: { [name: string]: Scope }; + parent?: Scope; + data: any; + + get(name: any, extraId: string): Scope; + + find(states): Scope | undefined; + + hasParent(states): boolean; + + count(): number; + + getAllScopeNames(): string[]; + + toStack(): any[]; + } +} declare module "./src/lib/keys" { export function keyCodeToString(keyCode: number): string; } diff --git a/demo/kitchen-sink/token_tooltip.js b/demo/kitchen-sink/token_tooltip.js index f352463aecb..4ce70fa8d3d 100644 --- a/demo/kitchen-sink/token_tooltip.js +++ b/demo/kitchen-sink/token_tooltip.js @@ -58,13 +58,25 @@ class TokenTooltip extends Tooltip { return; } - var tokenText = token.type; - if (token.state) - tokenText += "|" + token.state; - if (token.merge) - tokenText += "\n merge"; - if (token.stateTransitions) - tokenText += "\n " + token.stateTransitions.join("\n "); + var tokenText = ""; + var scope = token.type; + if (scope.name !== undefined) { + let firstScope = true; + do { + tokenText += firstScope ? scope.name + "\n" :" " + scope.name + "\n"; + firstScope = false; + if (!scope.parent) + tokenText += "\ntoken count:" + count(scope); + } while(scope = scope.parent) + } else { + tokenText += token.type; + if (token.state) + tokenText += "|" + token.state; + if (token.merge) + tokenText += "\n merge"; + if (token.stateTransitions) + tokenText += "\n " + token.stateTransitions.join("\n "); + } if (this.tokenText != tokenText) { this.setText(tokenText); @@ -121,4 +133,10 @@ class TokenTooltip extends Tooltip { } +function count(root) { + return Object.keys(root.children).reduce(function (n, key) { + return n + count(root.children[key]); + }, 1); +} + exports.TokenTooltip = TokenTooltip; diff --git a/src/background_tokenizer.js b/src/background_tokenizer.js index f96599350b8..b896921e2c8 100644 --- a/src/background_tokenizer.js +++ b/src/background_tokenizer.js @@ -193,8 +193,8 @@ class BackgroundTokenizer { var state = this.states[row - 1]; // @ts-expect-error TODO: potential wrong argument var data = this.tokenizer.getLineTokens(line, state, row); - - if (this.states[row] + "" !== data.state + "") { + + if (this.states[row] !== data.state) { this.states[row] = data.state; this.lines[row + 1] = null; if (this.currentLine > row + 1) diff --git a/src/edit_session/folding.js b/src/edit_session/folding.js index 2951d695c76..ee063f5ed83 100644 --- a/src/edit_session/folding.js +++ b/src/edit_session/folding.js @@ -710,7 +710,7 @@ function Folding() { if (token && /^comment|string/.test(type)) { type = type.match(/comment|string/)[0]; if (type == "comment") - type += "|doc-start|\\.doc"; + type += "|doc-start|\\.doc|empty"; var re = new RegExp(type); var range = new Range(); if (dir != 1) { diff --git a/src/layer/text_util.js b/src/layer/text_util.js index 97d197db877..03df39b4570 100644 --- a/src/layer/text_util.js +++ b/src/layer/text_util.js @@ -2,5 +2,5 @@ const textTokens = new Set(["text", "rparen", "lparen"]); exports.isTextToken = function(tokenType) { - return textTokens.has(tokenType); + return textTokens.has(tokenType.toString()); }; diff --git a/src/mode/folding/mixed.js b/src/mode/folding/mixed.js index 996584a62d5..683d65e1004 100644 --- a/src/mode/folding/mixed.js +++ b/src/mode/folding/mixed.js @@ -13,7 +13,7 @@ oop.inherits(FoldMode, BaseFoldMode); this.$getMode = function(state) { - if (typeof state != "string") + if (Array.isArray(state)) state = state[0]; for (var key in this.subModes) { if (state.indexOf(key) === 0) diff --git a/src/mode/lua_highlight_rules.js b/src/mode/lua_highlight_rules.js index b03286dc78b..7bf4f8dbca7 100644 --- a/src/mode/lua_highlight_rules.js +++ b/src/mode/lua_highlight_rules.js @@ -7,13 +7,13 @@ var LuaHighlightRules = function() { var keywords = ( "break|do|else|elseif|end|for|function|if|in|local|repeat|"+ - "return|then|until|while|or|and|not" + "return|then|until|while|or|and|not" ); var builtinConstants = ("true|false|nil|_G|_VERSION"); var functions = ( - // builtinFunctions + // builtinFunctions "string|xpcall|package|tostring|print|os|unpack|require|"+ "getfenv|setmetatable|next|assert|tonumber|io|rawequal|"+ "collectgarbage|getmetatable|module|rawset|math|debug|"+ @@ -34,9 +34,9 @@ var LuaHighlightRules = function() { "setupvalue|getlocal|getregistry|getfenv|setn|insert|getn|"+ "foreachi|maxn|foreach|concat|sort|remove|resume|yield|"+ "status|wrap|create|running|"+ - // metatableMethods + // metatableMethods "__add|__sub|__mod|__unm|__concat|__lt|__index|__call|__gc|__metatable|"+ - "__mul|__div|__pow|__len|__eq|__le|__newindex|__tostring|__mode|__tonumber" + "__mul|__div|__pow|__len|__eq|__le|__newindex|__tostring|__mode|__tonumber" ); var stdLibaries = ("string|package|os|io|math|debug|table|coroutine"); @@ -64,27 +64,24 @@ var LuaHighlightRules = function() { this.$rules = { "start" : [{ stateName: "bracketedComment", - onMatch : function(value, currentState, stack){ - stack.unshift(this.next, value.length - 2, currentState); - return "comment"; + onMatch2 : function(value, scope){ + var parent = scope.get("bracketedComment" + (value.length - 2)) + parent.meta = (value.length - 2); + return parent.get(this.next).get("comment"); }, regex : /\-\-\[=*\[/, next : [ { - onMatch : function(value, currentState, stack) { - if (value.length == stack[1]) { - stack.shift(); - stack.shift(); - this.next = stack.shift(); + onMatch2 : function(value, scope) { + if (scope.parent && value.length == scope.parent.meta) { + return scope.parent.parent.get("comment"); } else { - this.next = ""; + return scope.get("comment"); } - return "comment"; }, regex : /\]=*\]/, - next : "start" }, { - defaultToken: "comment.body" + defaultToken : "comment" } ] }, @@ -95,26 +92,23 @@ var LuaHighlightRules = function() { }, { stateName: "bracketedString", - onMatch : function(value, currentState, stack){ - stack.unshift(this.next, value.length, currentState); - return "string.start"; + onMatch2 : function(value, scope){ + var parent = scope.get("bracketedString" + value.length); + parent.meta = value.length; + return parent.get(this.next).get("string.start"); }, regex : /\[=*\[/, next : [ { - onMatch : function(value, currentState, stack) { - if (value.length == stack[1]) { - stack.shift(); - stack.shift(); - this.next = stack.shift(); + onMatch2 : function(value, scope) { + if (scope.parent && value.length == scope.parent.meta) { + return scope.parent.parent.get("string.end"); } else { - this.next = ""; + return scope.get("string.end"); } - return "string.end"; }, - + regex : /\]=*\]/, - next : "start" }, { defaultToken : "string" } @@ -149,7 +143,7 @@ var LuaHighlightRules = function() { regex : "\\s+|\\w+" } ] }; - + this.normalizeRules(); }; diff --git a/src/mode/text.js b/src/mode/text.js index 7bdc201a8ee..8f9f54b9b3c 100644 --- a/src/mode/text.js +++ b/src/mode/text.js @@ -32,7 +32,20 @@ Mode = function() { this.getTokenizer = function() { if (!this.$tokenizer) { this.$highlightRules = this.$highlightRules || new this.HighlightRules(this.$highlightRuleConfig); - this.$tokenizer = new Tokenizer(this.$highlightRules.getRules()); + var modeName; + if (this.$id) { + var chunks = this.$id.split('/'); + if (chunks.length > 1) { + modeName = chunks[chunks.length - 1]; + } + else { + modeName = chunks; + } + } + else { + modeName = "root"; + } + this.$tokenizer = new Tokenizer(this.$highlightRules.getRules(), modeName); } return this.$tokenizer; }; diff --git a/src/mode/text_highlight_rules.js b/src/mode/text_highlight_rules.js index a866846445e..4b99d3679d1 100644 --- a/src/mode/text_highlight_rules.js +++ b/src/mode/text_highlight_rules.js @@ -99,12 +99,16 @@ TextHighlightRules = function() { }; var pushState = function(currentState, stack) { - if (currentState != "start" || stack.length) - stack.unshift(this.nextState, currentState); + if (typeof currentState.get == "function") { + return currentState.get(this.nextState); + } + if (currentState !== "start" || stack.length) stack.unshift(this.nextState, currentState); return this.nextState; }; var popState = function(currentState, stack) { - // if (stack[0] === currentState) + if (typeof currentState.get == "function") { + return (currentState.parent.parent) ? currentState.parent : currentState.get("start"); + } stack.shift(); return stack.shift() || "start"; }; diff --git a/src/scope.js b/src/scope.js new file mode 100644 index 00000000000..4a892527287 --- /dev/null +++ b/src/scope.js @@ -0,0 +1,94 @@ +"use strict"; + +class Scope { + /** + * @param {any} name + * @param {Scope} [parent] + */ + constructor(name, parent) { + /**@type {Scope & String}*/ + // @ts-ignore + // eslint-disable-next-line no-new-wrappers + let stringObj = new String(name); //TODO: can be removed when ace builds switches to classes + stringObj.name = stringObj.toString(); // Assign the string representation to `name` + stringObj.children = {}; // Initialize the children object + stringObj.parent = parent; // Set the parent scope + stringObj.data = name; // Store the original data + + stringObj.get = this.get; + stringObj.find = this.find; + stringObj.hasParent = this.hasParent; + stringObj.count = this.count; + stringObj.getAllScopeNames = this.getAllScopeNames; + stringObj.toStack = this.toStack; + + return stringObj; + } + + /** + * @param {any} name + * @param {string|undefined} [extraId] + */ + get(name, extraId) { + var id = "" + name + (extraId || ""); + if (this.children[id]) { + return this.children[id]; + } + this.children[id] = new Scope(name, this); + if (extraId) { + this.children[id].data = extraId; + } + return this.children[id]; + } + + find(states) { + /**@type{Scope}*/ + var s = this; + while (s && !states[s.name]) { + s = s.parent; + } + return states[s ? s.name : "start"]; + } + + hasParent(states) { + /**@type{Scope}*/ + var s = this; + while (s && states !== s.name) { + s = s.parent; + } + return s != undefined; + } + + count() { + var s = 1; + for (var i in this.children) s += this.children[i].count(); + return s; + } + + /** + * + * @returns {string[]} + */ + getAllScopeNames() { + var scopeNames = []; + /**@type{Scope}*/ + var self = this; + do { + scopeNames.push(self.name); + } while (self = self.parent); + return scopeNames; + } + + toStack() { + var stack = []; + /**@type{Scope}*/ + var self = this; + do { + stack.push(self.data); + } while (self = self.parent); + return stack; + } +} + + +exports.Scope = Scope; diff --git a/src/tokenizer.js b/src/tokenizer.js index 900651d7ca3..3438cf9458d 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -1,6 +1,7 @@ "use strict"; const reportError = require("./lib/report_error").reportError; +var {Scope} = require("./scope"); // tokenizing lines longer than this makes editor very slow var MAX_TOKEN_COUNT = 2000; /** @@ -10,11 +11,13 @@ class Tokenizer { /** * Constructs a new tokenizer based on the given rules and flags. * @param {Object} rules The highlighting rules - **/ - constructor(rules) { + * @param {string} [modeName] + */ + constructor(rules, modeName) { /**@type {RegExp}*/ this.splitRegex; this.states = rules; + this.rootScope = new Scope(modeName || "root"); this.regExps = {}; this.matchMappings = {}; @@ -203,36 +206,40 @@ class Tokenizer { if (lastCapture.end != null && /^\)*$/.test(src.substr(lastCapture.end))) src = src.substring(0, lastCapture.start) + src.substr(lastCapture.end); } - + // this is needed for regexps that can match in multiple ways if (src.charAt(0) != "^") src = "^" + src; if (src.charAt(src.length - 1) != "$") src += "$"; - + return new RegExp(src, (flag||"").replace("g", "")); } /** * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state. - * @param {string} line - * @param {string | string[]} startState - * @returns {{tokens:import("../ace-internal").Ace.Token[], state: string|string[]}} - */ + * @param {String} line + * @param {String|Scope} [startState] + * @returns {Object} + **/ getLineTokens(line, startState) { - if (startState && typeof startState != "string") { - /**@type {any[]}*/ - var stack = startState.slice(0); - startState = stack[0]; - if (startState === "#tmp") { + if (startState && startState["getAllScopeNames"]) { + var stack = /**@type{Scope}*/(startState).toStack(); + stack.pop(); //drop the root scope name + + if (/**@type{Scope}*/(startState).data === "#tmp") { stack.shift(); - startState = stack.shift(); } - } else + if (stack.length === 1) { + stack = []; + } + } + else { var stack = []; - - var currentState = /**@type{string}*/(startState) || "start"; + } + var currentState = (startState !== undefined) ? (startState["getAllScopeNames"]) ? /**@type{Scope}*/(startState) : this.rootScope.get( + startState) : this.rootScope.get("start"); var state = this.states[currentState]; if (!state) { - currentState = "start"; + currentState = this.rootScope.get("start"); state = this.states[currentState]; } var mapping = this.matchMappings[currentState]; @@ -243,47 +250,74 @@ class Tokenizer { var lastIndex = 0; var matchAttempts = 0; - var token = {type: null, value: ""}; + var token = { + type: null, + value: "" + }; while (match = re.exec(line)) { var type = mapping.defaultToken; var rule = null; + var rememberedState = undefined; var value = match[0]; var index = re.lastIndex; if (index - value.length > lastIndex) { var skipped = line.substring(lastIndex, index - value.length); - if (token.type == type) { + if (token.type && token.type === type) { token.value += skipped; - } else { - if (token.type) - tokens.push(token); - token = {type: type, value: skipped}; + } + else { + if (token.type) tokens.push(token); + token = { + type: currentState.get(type), + value: skipped + }; } } - for (var i = 0; i < match.length-2; i++) { - if (match[i + 1] === undefined) - continue; + for (var i = 0; i < match.length - 2; i++) { + if (match[i + 1] === undefined) continue; rule = state[mapping[i]]; - if (rule.onMatch) - type = rule.onMatch(value, currentState, stack, line); - else - type = rule.token; + if (rule.onMatch || rule.onMatch2) { + if (rule.onMatch) { + type = rule.onMatch(value, currentState.toString(), stack, line); + } + else { + type = rule.onMatch2(value, currentState, line); + rememberedState = (Array.isArray(type)) ? type[0].type.parent : type.parent; + } - if (rule.next) { - if (typeof rule.next == "string") { - currentState = rule.next; - } else { - currentState = rule.next(currentState, stack); + } + else { + if (rule.token) type = rule.token; + } + + if (rule.next || rule.next2 || rememberedState) { + if (!rememberedState) { + if (typeof rule.next !== 'function') { + currentState = currentState.parent.get(rule.next); + } + else { + if (rule.next) { + currentState = rule.next(currentState.toString(), stack); + currentState = this.fromStack(stack, currentState); + } + else { + currentState = rule.next2(currentState, stack); + } + } } - + else { + currentState = rememberedState; + } + state = this.states[currentState]; if (!state) { this.reportError("state doesn't exist", currentState); - currentState = "start"; + currentState = this.rootScope.get("start"); state = this.states[currentState]; } mapping = this.matchMappings[currentState]; @@ -291,31 +325,41 @@ class Tokenizer { re = this.regExps[currentState]; re.lastIndex = index; } - if (rule.consumeLineEnd) - lastIndex = index; + if (rule.consumeLineEnd) lastIndex = index; break; } if (value) { - if (typeof type === "string") { + if (type && !type["getAllScopeNames"]) { + currentState = this.fromStack(stack, currentState.toString()); + } + + if (!Array.isArray(type)) { if ((!rule || rule.merge !== false) && token.type === type) { token.value += value; - } else { - if (token.type) - tokens.push(token); - token = {type: type, value: value}; } - } else if (type) { - if (token.type) - tokens.push(token); - token = {type: null, value: ""}; - for (var i = 0; i < type.length; i++) + else { + if (token.type) tokens.push(token); + token = { + type: currentState.get(type), + value: value + }; + } + } + else if (type) { + if (token.type) tokens.push(token); + token = { + type: null, + value: "" + }; + for (var i = 0; i < type.length; i++) { + type[i].type = currentState.get(type[i].type); tokens.push(type[i]); + } } } - if (lastIndex == line.length) - break; + if (lastIndex === line.length) break; lastIndex = index; @@ -328,31 +372,55 @@ class Tokenizer { } // chrome doens't show contents of text nodes with very long text while (lastIndex < line.length) { - if (token.type) - tokens.push(token); + if (token.type) tokens.push(token); token = { value: line.substring(lastIndex, lastIndex += 500), - type: "overflow" + type: currentState.get("overflow") }; } - currentState = "start"; + currentState = this.rootScope.get("start"); stack = []; break; } } - if (token.type) + if (token.type) tokens.push(token); + + if (!tokens.length || tokens[tokens.length - 1].type.parent !== currentState) { + token = { + value: "", + type: currentState.get("empty") + }; tokens.push(token); - - if (stack.length > 1) { - if (stack[0] !== currentState) - stack.unshift("#tmp", currentState); } + return { - tokens : tokens, - state : stack.length ? stack : currentState + tokens: tokens, + state: currentState }; } + + /** + * Retrieves the scope for the given stack and current state. + * + * @param {string[]} stack - The stack of scopes. + * @param {string} currentState - The current state. + * @returns {Scope} The scope for the given stack and current state. + */ + fromStack(stack, currentState) { + let scope = this.rootScope; + if (stack.length === 0) { + return scope.get(currentState); //the start state + } + for (var i = stack.length - 1; i >= 0; i--) { + scope = scope.get(stack[i]); + } + if (stack[0] !== currentState) { + scope = scope.get(currentState, "#tmp"); + } + + return scope; + } } Tokenizer.prototype.reportError = reportError; diff --git a/src/tooltip.js b/src/tooltip.js index f0f69951739..67cc6ab6c2c 100644 --- a/src/tooltip.js +++ b/src/tooltip.js @@ -56,8 +56,8 @@ class Tooltip { * @param {Number} y **/ setPosition(x, y) { - this.getElement().style.left = x + "px"; - this.getElement().style.top = y + "px"; + this.getElement().style.left = Math.round(x) + "px"; + this.getElement().style.top = Math.round(y) + "px"; } /** From fb48defe999244696b43f265e9dbe1e3d8ea8db7 Mon Sep 17 00:00:00 2001 From: mkslanc Date: Mon, 2 Sep 2024 16:42:16 +0400 Subject: [PATCH 2/6] changes to make snippets tokenizer work --- src/tokenizer.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/tokenizer.js b/src/tokenizer.js index 3438cf9458d..49d4b6ffdd7 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -235,7 +235,7 @@ class Tokenizer { else { var stack = []; } - var currentState = (startState !== undefined) ? (startState["getAllScopeNames"]) ? /**@type{Scope}*/(startState) : this.rootScope.get( + var currentState = (startState != undefined) ? (startState["getAllScopeNames"]) ? /**@type{Scope}*/(startState) : this.rootScope.get( startState) : this.rootScope.get("start"); var state = this.states[currentState]; if (!state) { @@ -268,7 +268,7 @@ class Tokenizer { token.value += skipped; } else { - if (token.type) tokens.push(token); + if (token.type && token.type != "") tokens.push(token); token = { type: currentState.get(type), value: skipped @@ -292,7 +292,7 @@ class Tokenizer { } else { - if (rule.token) type = rule.token; + if (rule.token != undefined) type = rule.token; } if (rule.next || rule.next2 || rememberedState) { @@ -334,7 +334,7 @@ class Tokenizer { currentState = this.fromStack(stack, currentState.toString()); } - if (!Array.isArray(type)) { + if (type && !Array.isArray(type) && type != "") { if ((!rule || rule.merge !== false) && token.type === type) { token.value += value; } @@ -353,7 +353,9 @@ class Tokenizer { value: "" }; for (var i = 0; i < type.length; i++) { + if (type[i].type) { type[i].type = currentState.get(type[i].type); + } tokens.push(type[i]); } } @@ -386,7 +388,7 @@ class Tokenizer { if (token.type) tokens.push(token); - if (!tokens.length || tokens[tokens.length - 1].type.parent !== currentState) { + if (!tokens.length || tokens[tokens.length - 1].type && tokens[tokens.length - 1].type.parent !== currentState) { token = { value: "", type: currentState.get("empty") From bf581adbe599a7316c7fce47933b5e2d5f657c8c Mon Sep 17 00:00:00 2001 From: mkslanc Date: Mon, 2 Sep 2024 18:11:21 +0400 Subject: [PATCH 3/6] refactor due to code review --- src/mode/lua_highlight_rules.js | 2 +- src/mode/text.js | 8 +------- src/scope.js | 28 ++++++++++++++++++++++++++++ src/tokenizer.js | 24 ++---------------------- src/tooltip.js | 4 ++-- 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/mode/lua_highlight_rules.js b/src/mode/lua_highlight_rules.js index 7bf4f8dbca7..2be37fe1a1f 100644 --- a/src/mode/lua_highlight_rules.js +++ b/src/mode/lua_highlight_rules.js @@ -81,7 +81,7 @@ var LuaHighlightRules = function() { }, regex : /\]=*\]/, }, { - defaultToken : "comment" + defaultToken : "comment.body" } ] }, diff --git a/src/mode/text.js b/src/mode/text.js index 8f9f54b9b3c..f775e897c18 100644 --- a/src/mode/text.js +++ b/src/mode/text.js @@ -34,13 +34,7 @@ Mode = function() { this.$highlightRules = this.$highlightRules || new this.HighlightRules(this.$highlightRuleConfig); var modeName; if (this.$id) { - var chunks = this.$id.split('/'); - if (chunks.length > 1) { - modeName = chunks[chunks.length - 1]; - } - else { - modeName = chunks; - } + modeName = this.$id.split('/').pop(); } else { modeName = "root"; diff --git a/src/scope.js b/src/scope.js index 4a892527287..2f3db3caba8 100644 --- a/src/scope.js +++ b/src/scope.js @@ -21,6 +21,7 @@ class Scope { stringObj.count = this.count; stringObj.getAllScopeNames = this.getAllScopeNames; stringObj.toStack = this.toStack; + stringObj.fromStack = this.fromStack; return stringObj; } @@ -88,6 +89,33 @@ class Scope { } while (self = self.parent); return stack; } + + /** + * Retrieves the scope for the given stack and current state. + * + * @param {string[]} stack - The stack of scopes. + * @param {string | Scope} currentState - The current state. + * @returns {Scope} The scope for the given stack and current state. + */ + fromStack(stack, currentState) { + /**@type{Scope}*/ + let scope = this; + while (scope.parent) { + scope = scope.parent; + } + + if (stack.length === 0) { + return scope.get(currentState); //the start state + } + for (var i = stack.length - 1; i >= 0; i--) { + scope = scope.get(stack[i]); + } + if (stack[0] !== currentState) { + scope = scope.get(currentState, "#tmp"); + } + + return scope; + } } diff --git a/src/tokenizer.js b/src/tokenizer.js index 49d4b6ffdd7..5f2dd7d5956 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -303,7 +303,7 @@ class Tokenizer { else { if (rule.next) { currentState = rule.next(currentState.toString(), stack); - currentState = this.fromStack(stack, currentState); + currentState = this.rootScope.fromStack(stack, currentState); } else { currentState = rule.next2(currentState, stack); @@ -331,7 +331,7 @@ class Tokenizer { if (value) { if (type && !type["getAllScopeNames"]) { - currentState = this.fromStack(stack, currentState.toString()); + currentState = this.rootScope.fromStack(stack, currentState); } if (type && !Array.isArray(type) && type != "") { @@ -402,27 +402,7 @@ class Tokenizer { }; } - /** - * Retrieves the scope for the given stack and current state. - * - * @param {string[]} stack - The stack of scopes. - * @param {string} currentState - The current state. - * @returns {Scope} The scope for the given stack and current state. - */ - fromStack(stack, currentState) { - let scope = this.rootScope; - if (stack.length === 0) { - return scope.get(currentState); //the start state - } - for (var i = stack.length - 1; i >= 0; i--) { - scope = scope.get(stack[i]); - } - if (stack[0] !== currentState) { - scope = scope.get(currentState, "#tmp"); - } - return scope; - } } Tokenizer.prototype.reportError = reportError; diff --git a/src/tooltip.js b/src/tooltip.js index 67cc6ab6c2c..f0f69951739 100644 --- a/src/tooltip.js +++ b/src/tooltip.js @@ -56,8 +56,8 @@ class Tooltip { * @param {Number} y **/ setPosition(x, y) { - this.getElement().style.left = Math.round(x) + "px"; - this.getElement().style.top = Math.round(y) + "px"; + this.getElement().style.left = x + "px"; + this.getElement().style.top = y + "px"; } /** From 57dd6b7d53e3643552adc83405b710796a18bbb2 Mon Sep 17 00:00:00 2001 From: mkslanc Date: Sat, 7 Sep 2024 15:10:43 +0400 Subject: [PATCH 4/6] fix: strict equal checks/tests # Conflicts: # src/mode/lua_highlight_rules.js --- src/autocomplete/inline_test.js | 4 ++-- src/ext/beautify.js | 28 ++++++++++++++-------------- src/ext/simple_tokenizer_test.js | 4 ++++ src/mode/lua_highlight_rules.js | 16 ++++++---------- src/tokenizer.js | 8 ++++---- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/autocomplete/inline_test.js b/src/autocomplete/inline_test.js index c1cc1b07020..a717446ea2b 100644 --- a/src/autocomplete/inline_test.js +++ b/src/autocomplete/inline_test.js @@ -344,7 +344,7 @@ module.exports = { // Text to the right of the cursor should be tokenized normally again. var tokens = editor.session.getTokens(2); assert.strictEqual(tokens[0].value, "f hi I should be hidden"); - assert.strictEqual(tokens[0].type, "text"); + assert.equal(tokens[0].type, "text"); done(); }, @@ -375,7 +375,7 @@ module.exports = { // Text to the right of the cursor should be tokenized normally again. var tokens = editor.session.getTokens(2); assert.strictEqual(tokens[0].value, "fhi I should be hidden"); - assert.strictEqual(tokens[0].type, "text"); + assert.equal(tokens[0].type, "text"); done(); }, diff --git a/src/ext/beautify.js b/src/ext/beautify.js index be367fb4e21..c6b83bb3c9e 100644 --- a/src/ext/beautify.js +++ b/src/ext/beautify.js @@ -127,7 +127,7 @@ exports.beautify = function(session) { } // line break before } - if (!inTag && !rowsToAdd && token.type === "paren.rparen" && token.value.substr(0, 1) === "}") { + if (!inTag && !rowsToAdd && token.type == "paren.rparen" && token.value.substr(0, 1) === "}") { rowsToAdd++; } @@ -153,7 +153,7 @@ exports.beautify = function(session) { if (value) { // whitespace - if (token.type === "keyword" && value.match(/^(if|else|elseif|for|foreach|while|switch)$/)) { + if (token.type == "keyword" && value.match(/^(if|else|elseif|for|foreach|while|switch)$/)) { parents[depth] = value; trimNext(); @@ -167,7 +167,7 @@ exports.beautify = function(session) { } } // trim value after opening paren - } else if (token.type === "paren.lparen") { + } else if (token.type == "paren.lparen") { trimNext(); // whitespace after { @@ -194,7 +194,7 @@ exports.beautify = function(session) { } } // remove space before closing paren - } else if (token.type === "paren.rparen") { + } else if (token.type == "paren.rparen") { unindent = 1; // ensure curly brace is preceeded by whitespace @@ -232,13 +232,13 @@ exports.beautify = function(session) { trimLine(); // add spaces around conditional operators - } else if ((token.type === "keyword.operator" || token.type === "keyword") && value.match(/^(=|==|===|!=|!==|&&|\|\||and|or|xor|\+=|.=|>|>=|<|<=|=>)$/)) { + } else if ((token.type == "keyword.operator" || token.type == "keyword") && value.match(/^(=|==|===|!=|!==|&&|\|\||and|or|xor|\+=|.=|>|>=|<|<=|=>)$/)) { trimCode(); trimNext(); spaceBefore = true; spaceAfter = true; // remove space before semicolon - } else if (token.type === "punctuation.operator" && value === ';') { + } else if (token.type == "punctuation.operator" && value === ';') { trimCode(); trimNext(); spaceAfter = true; @@ -246,7 +246,7 @@ exports.beautify = function(session) { if (inCSS) rowsToAdd++; // space after colon or comma - } else if (token.type === "punctuation.operator" && value.match(/^(:|,)$/)) { + } else if (token.type == "punctuation.operator" && value.match(/^(:|,)$/)) { trimCode(); trimNext(); @@ -258,7 +258,7 @@ exports.beautify = function(session) { breakBefore = false; } // ensure space before php closing tag - } else if (token.type === "support.php_tag" && value === "?>" && !breakBefore) { + } else if (token.type == "support.php_tag" && value === "?>" && !breakBefore) { trimCode(); spaceBefore = true; // remove excess space before HTML attribute @@ -273,7 +273,7 @@ exports.beautify = function(session) { trimLine(); if(value === "/>") spaceBefore = true; - } else if (token.type === "keyword" && value.match(/^(case|default)$/)) { + } else if (token.type == "keyword" && value.match(/^(case|default)$/)) { if (caseBody) unindent = 1; } @@ -306,13 +306,13 @@ exports.beautify = function(session) { code += tabString; } - if (token.type === "keyword" && value.match(/^(case|default)$/)) { + if (token.type == "keyword" && value.match(/^(case|default)$/)) { if (caseBody === false) { parents[depth] = value; depth++; caseBody = true; } - } else if (token.type === "keyword" && value.match(/^(break)$/)) { + } else if (token.type == "keyword" && value.match(/^(break)$/)) { if(parents[depth-1] && parents[depth-1].match(/^(case|default)$/)) { depth--; caseBody = false; @@ -320,19 +320,19 @@ exports.beautify = function(session) { } // indent one line after if or else - if (token.type === "paren.lparen") { + if (token.type == "paren.lparen") { roundDepth += (value.match(/\(/g) || []).length; curlyDepth += (value.match(/\{/g) || []).length; depth += value.length; } - if (token.type === "keyword" && value.match(/^(if|else|elseif|for|while)$/)) { + if (token.type == "keyword" && value.match(/^(if|else|elseif|for|while)$/)) { indentNextLine = true; roundDepth = 0; } else if (!roundDepth && value.trim() && token.type !== "comment") indentNextLine = false; - if (token.type === "paren.rparen") { + if (token.type == "paren.rparen") { roundDepth -= (value.match(/\)/g) || []).length; curlyDepth -= (value.match(/\}/g) || []).length; diff --git a/src/ext/simple_tokenizer_test.js b/src/ext/simple_tokenizer_test.js index d60183ce16b..6f5a71446ad 100644 --- a/src/ext/simple_tokenizer_test.js +++ b/src/ext/simple_tokenizer_test.js @@ -69,3 +69,7 @@ module.exports = { } }; + +if (typeof module !== "undefined" && module === require.main) { + require("asyncjs").test.testcase(module.exports).exec(); +} diff --git a/src/mode/lua_highlight_rules.js b/src/mode/lua_highlight_rules.js index 2be37fe1a1f..2e1bacebc10 100644 --- a/src/mode/lua_highlight_rules.js +++ b/src/mode/lua_highlight_rules.js @@ -65,16 +65,14 @@ var LuaHighlightRules = function() { "start" : [{ stateName: "bracketedComment", onMatch2 : function(value, scope){ - var parent = scope.get("bracketedComment" + (value.length - 2)) - parent.meta = (value.length - 2); - return parent.get(this.next).get("comment"); + return scope.get(this.next, value.length - 2).get("comment"); }, regex : /\-\-\[=*\[/, next : [ { onMatch2 : function(value, scope) { - if (scope.parent && value.length == scope.parent.meta) { - return scope.parent.parent.get("comment"); + if (scope == "bracketedComment" && value.length == scope.data) { + return scope.parent.get("comment"); } else { return scope.get("comment"); } @@ -93,16 +91,14 @@ var LuaHighlightRules = function() { { stateName: "bracketedString", onMatch2 : function(value, scope){ - var parent = scope.get("bracketedString" + value.length); - parent.meta = value.length; - return parent.get(this.next).get("string.start"); + return scope.get(this.next, value.length - 2).get("string.start"); }, regex : /\[=*\[/, next : [ { onMatch2 : function(value, scope) { - if (scope.parent && value.length == scope.parent.meta) { - return scope.parent.parent.get("string.end"); + if (scope == "bracketedString" && value.length == scope.data) { + return scope.parent.get("string.end"); } else { return scope.get("string.end"); } diff --git a/src/tokenizer.js b/src/tokenizer.js index 5f2dd7d5956..dac549d52b1 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -264,7 +264,7 @@ class Tokenizer { if (index - value.length > lastIndex) { var skipped = line.substring(lastIndex, index - value.length); - if (token.type && token.type === type) { + if (token.type && token.type == type) { token.value += skipped; } else { @@ -297,7 +297,7 @@ class Tokenizer { if (rule.next || rule.next2 || rememberedState) { if (!rememberedState) { - if (typeof rule.next !== 'function') { + if (rule.next && typeof rule.next !== 'function') { currentState = currentState.parent.get(rule.next); } else { @@ -306,7 +306,7 @@ class Tokenizer { currentState = this.rootScope.fromStack(stack, currentState); } else { - currentState = rule.next2(currentState, stack); + currentState = rule.next2(currentState); } } } @@ -335,7 +335,7 @@ class Tokenizer { } if (type && !Array.isArray(type) && type != "") { - if ((!rule || rule.merge !== false) && token.type === type) { + if ((!rule || rule.merge !== false) && token.type == type) { token.value += value; } else { From c6141fad4af8b44508369c7431c43f6f1409e1e5 Mon Sep 17 00:00:00 2001 From: mkslanc Date: Sat, 7 Sep 2024 15:14:38 +0400 Subject: [PATCH 5/6] make it work with scopes --- src/mode/_test/highlight_rules_test.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/mode/_test/highlight_rules_test.js b/src/mode/_test/highlight_rules_test.js index 9bbab769557..906771a3883 100644 --- a/src/mode/_test/highlight_rules_test.js +++ b/src/mode/_test/highlight_rules_test.js @@ -104,9 +104,9 @@ function checkModes() { var str = blockComment.start + " " + blockComment.end; str = blockComment.start + str; if (blockComment.nestable) - str += blockComment.end; + str += blockComment.end; var data = tokenizer.getLineTokens(str, "start"); - var isBroken = data.tokens.some(function(t) { return !/comment/.test(t.type); }); + var isBroken = data.tokens.some(function(t) { return !/comment|empty/.test(t.type); }); if (isBroken) { die("broken blockComment in " + modeName, data); } @@ -224,7 +224,9 @@ function generateTestData(names, force) { var tokenizedLine = ""; data.tokens.forEach(function(x) { tokenizedLine += x.value; - tmp.push(JSON.stringify([x.type, x.value])); + if (x.type != "empty") { + tmp.push(JSON.stringify([x.type, x.value])); + } }); if (tokenizedLine != line) tmp.push(JSON.stringify(line)); @@ -278,6 +280,7 @@ function testMode(modeName, i) { line = lineData.values.join(""); var tokens = tokenizer.getLineTokens(line, state); + tokens.tokens = tokens.tokens.filter((el) => el.type != "empty"); var values = tokens.tokens.map(function(x) {return x.value;}); var types = tokens.tokens.map(function(x) {return x.type;}); From 8839b0f94004b8363114e1a27d999caa9633d5d5 Mon Sep 17 00:00:00 2001 From: mkslanc Date: Tue, 17 Jun 2025 17:56:24 +0400 Subject: [PATCH 6/6] refactor: simplify scope stack manipulation and fix related issues --- ace-internal.d.ts | 7 +------ src/mode/lua_highlight_rules.js | 2 +- src/scope.js | 12 +++++++++++- src/tokenizer.js | 12 ++++-------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/ace-internal.d.ts b/ace-internal.d.ts index e9a6f9a5bf0..582946cc3b2 100644 --- a/ace-internal.d.ts +++ b/ace-internal.d.ts @@ -1650,9 +1650,4 @@ declare module "./src/scope" { toStack(): any[]; } -} - -declare module "./src/lib/keys" { - export function keyCodeToString(keyCode: number): string; -} - +} \ No newline at end of file diff --git a/src/mode/lua_highlight_rules.js b/src/mode/lua_highlight_rules.js index 2e1bacebc10..eba8b29a61b 100644 --- a/src/mode/lua_highlight_rules.js +++ b/src/mode/lua_highlight_rules.js @@ -91,7 +91,7 @@ var LuaHighlightRules = function() { { stateName: "bracketedString", onMatch2 : function(value, scope){ - return scope.get(this.next, value.length - 2).get("string.start"); + return scope.get(this.next, value.length).get("string.start"); }, regex : /\[=*\[/, next : [ diff --git a/src/scope.js b/src/scope.js index 2f3db3caba8..eef526d0739 100644 --- a/src/scope.js +++ b/src/scope.js @@ -87,6 +87,16 @@ class Scope { do { stack.push(self.data); } while (self = self.parent); + + stack.pop(); //drop the root scope name + + if (this.data === "#tmp") { + stack.shift(); + } + if (stack.length === 1) { + stack = []; + } + return stack; } @@ -110,7 +120,7 @@ class Scope { for (var i = stack.length - 1; i >= 0; i--) { scope = scope.get(stack[i]); } - if (stack[0] !== currentState) { + if (stack[0] != currentState) { scope = scope.get(currentState, "#tmp"); } diff --git a/src/tokenizer.js b/src/tokenizer.js index dac549d52b1..64f75ec8d8b 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -223,14 +223,6 @@ class Tokenizer { getLineTokens(line, startState) { if (startState && startState["getAllScopeNames"]) { var stack = /**@type{Scope}*/(startState).toStack(); - stack.pop(); //drop the root scope name - - if (/**@type{Scope}*/(startState).data === "#tmp") { - stack.shift(); - } - if (stack.length === 1) { - stack = []; - } } else { var stack = []; @@ -307,11 +299,15 @@ class Tokenizer { } else { currentState = rule.next2(currentState); + + stack = currentState.toStack(); } } } else { currentState = rememberedState; + + stack = currentState.toStack(); } state = this.states[currentState];