diff --git a/ace-internal.d.ts b/ace-internal.d.ts index f5432c940c2..582946cc3b2 100644 --- a/ace-internal.d.ts +++ b/ace-internal.d.ts @@ -1257,7 +1257,7 @@ export namespace Ace { trim?: boolean, firstLineNumber?: number, showGutter?: boolean - } +} export interface Operation { command: { @@ -1630,3 +1630,24 @@ declare module "./src/mouse/default_gutter_handler" { } } +declare module "./src/scope" { + + export interface Scope extends String { + name: string; + children: { [name: string]: Scope }; + parent?: Scope; + data: any; + + get(name: any, extraId: string): Scope; + + find(states): Scope | undefined; + + hasParent(states): boolean; + + count(): number; + + getAllScopeNames(): string[]; + + toStack(): any[]; + } +} \ No newline at end of file diff --git a/demo/kitchen-sink/token_tooltip.js b/demo/kitchen-sink/token_tooltip.js index f352463aecb..4ce70fa8d3d 100644 --- a/demo/kitchen-sink/token_tooltip.js +++ b/demo/kitchen-sink/token_tooltip.js @@ -58,13 +58,25 @@ class TokenTooltip extends Tooltip { return; } - var tokenText = token.type; - if (token.state) - tokenText += "|" + token.state; - if (token.merge) - tokenText += "\n merge"; - if (token.stateTransitions) - tokenText += "\n " + token.stateTransitions.join("\n "); + var tokenText = ""; + var scope = token.type; + if (scope.name !== undefined) { + let firstScope = true; + do { + tokenText += firstScope ? scope.name + "\n" :" " + scope.name + "\n"; + firstScope = false; + if (!scope.parent) + tokenText += "\ntoken count:" + count(scope); + } while(scope = scope.parent) + } else { + tokenText += token.type; + if (token.state) + tokenText += "|" + token.state; + if (token.merge) + tokenText += "\n merge"; + if (token.stateTransitions) + tokenText += "\n " + token.stateTransitions.join("\n "); + } if (this.tokenText != tokenText) { this.setText(tokenText); @@ -121,4 +133,10 @@ class TokenTooltip extends Tooltip { } +function count(root) { + return Object.keys(root.children).reduce(function (n, key) { + return n + count(root.children[key]); + }, 1); +} + exports.TokenTooltip = TokenTooltip; diff --git a/src/autocomplete/inline_test.js b/src/autocomplete/inline_test.js index c1cc1b07020..a717446ea2b 100644 --- a/src/autocomplete/inline_test.js +++ b/src/autocomplete/inline_test.js @@ -344,7 +344,7 @@ module.exports = { // Text to the right of the cursor should be tokenized normally again. var tokens = editor.session.getTokens(2); assert.strictEqual(tokens[0].value, "f hi I should be hidden"); - assert.strictEqual(tokens[0].type, "text"); + assert.equal(tokens[0].type, "text"); done(); }, @@ -375,7 +375,7 @@ module.exports = { // Text to the right of the cursor should be tokenized normally again. var tokens = editor.session.getTokens(2); assert.strictEqual(tokens[0].value, "fhi I should be hidden"); - assert.strictEqual(tokens[0].type, "text"); + assert.equal(tokens[0].type, "text"); done(); }, diff --git a/src/background_tokenizer.js b/src/background_tokenizer.js index f62502aff8e..6666e795df3 100644 --- a/src/background_tokenizer.js +++ b/src/background_tokenizer.js @@ -193,8 +193,8 @@ class BackgroundTokenizer { var state = this.states[row - 1]; // @ts-expect-error TODO: potential wrong argument var data = this.tokenizer.getLineTokens(line, state, row); - - if (this.states[row] + "" !== data.state + "") { + + if (this.states[row] !== data.state) { this.states[row] = data.state; this.lines[row + 1] = null; if (this.currentLine > row + 1) diff --git a/src/edit_session/folding.js b/src/edit_session/folding.js index aaaeb916311..f1dc261f985 100644 --- a/src/edit_session/folding.js +++ b/src/edit_session/folding.js @@ -715,7 +715,7 @@ function Folding() { if (token && /^comment|string/.test(type)) { type = type.match(/comment|string/)[0]; if (type == "comment") - type += "|doc-start|\\.doc"; + type += "|doc-start|\\.doc|empty"; var re = new RegExp(type); var range = new Range(); if (dir != 1) { diff --git a/src/ext/beautify.js b/src/ext/beautify.js index be367fb4e21..c6b83bb3c9e 100644 --- a/src/ext/beautify.js +++ b/src/ext/beautify.js @@ -127,7 +127,7 @@ exports.beautify = function(session) { } // line break before } - if (!inTag && !rowsToAdd && token.type === "paren.rparen" && token.value.substr(0, 1) === "}") { + if (!inTag && !rowsToAdd && token.type == "paren.rparen" && token.value.substr(0, 1) === "}") { rowsToAdd++; } @@ -153,7 +153,7 @@ exports.beautify = function(session) { if (value) { // whitespace - if (token.type === "keyword" && value.match(/^(if|else|elseif|for|foreach|while|switch)$/)) { + if (token.type == "keyword" && value.match(/^(if|else|elseif|for|foreach|while|switch)$/)) { parents[depth] = value; trimNext(); @@ -167,7 +167,7 @@ exports.beautify = function(session) { } } // trim value after opening paren - } else if (token.type === "paren.lparen") { + } else if (token.type == "paren.lparen") { trimNext(); // whitespace after { @@ -194,7 +194,7 @@ exports.beautify = function(session) { } } // remove space before closing paren - } else if (token.type === "paren.rparen") { + } else if (token.type == "paren.rparen") { unindent = 1; // ensure curly brace is preceeded by whitespace @@ -232,13 +232,13 @@ exports.beautify = function(session) { trimLine(); // add spaces around conditional operators - } else if ((token.type === "keyword.operator" || token.type === "keyword") && value.match(/^(=|==|===|!=|!==|&&|\|\||and|or|xor|\+=|.=|>|>=|<|<=|=>)$/)) { + } else if ((token.type == "keyword.operator" || token.type == "keyword") && value.match(/^(=|==|===|!=|!==|&&|\|\||and|or|xor|\+=|.=|>|>=|<|<=|=>)$/)) { trimCode(); trimNext(); spaceBefore = true; spaceAfter = true; // remove space before semicolon - } else if (token.type === "punctuation.operator" && value === ';') { + } else if (token.type == "punctuation.operator" && value === ';') { trimCode(); trimNext(); spaceAfter = true; @@ -246,7 +246,7 @@ exports.beautify = function(session) { if (inCSS) rowsToAdd++; // space after colon or comma - } else if (token.type === "punctuation.operator" && value.match(/^(:|,)$/)) { + } else if (token.type == "punctuation.operator" && value.match(/^(:|,)$/)) { trimCode(); trimNext(); @@ -258,7 +258,7 @@ exports.beautify = function(session) { breakBefore = false; } // ensure space before php closing tag - } else if (token.type === "support.php_tag" && value === "?>" && !breakBefore) { + } else if (token.type == "support.php_tag" && value === "?>" && !breakBefore) { trimCode(); spaceBefore = true; // remove excess space before HTML attribute @@ -273,7 +273,7 @@ exports.beautify = function(session) { trimLine(); if(value === "/>") spaceBefore = true; - } else if (token.type === "keyword" && value.match(/^(case|default)$/)) { + } else if (token.type == "keyword" && value.match(/^(case|default)$/)) { if (caseBody) unindent = 1; } @@ -306,13 +306,13 @@ exports.beautify = function(session) { code += tabString; } - if (token.type === "keyword" && value.match(/^(case|default)$/)) { + if (token.type == "keyword" && value.match(/^(case|default)$/)) { if (caseBody === false) { parents[depth] = value; depth++; caseBody = true; } - } else if (token.type === "keyword" && value.match(/^(break)$/)) { + } else if (token.type == "keyword" && value.match(/^(break)$/)) { if(parents[depth-1] && parents[depth-1].match(/^(case|default)$/)) { depth--; caseBody = false; @@ -320,19 +320,19 @@ exports.beautify = function(session) { } // indent one line after if or else - if (token.type === "paren.lparen") { + if (token.type == "paren.lparen") { roundDepth += (value.match(/\(/g) || []).length; curlyDepth += (value.match(/\{/g) || []).length; depth += value.length; } - if (token.type === "keyword" && value.match(/^(if|else|elseif|for|while)$/)) { + if (token.type == "keyword" && value.match(/^(if|else|elseif|for|while)$/)) { indentNextLine = true; roundDepth = 0; } else if (!roundDepth && value.trim() && token.type !== "comment") indentNextLine = false; - if (token.type === "paren.rparen") { + if (token.type == "paren.rparen") { roundDepth -= (value.match(/\)/g) || []).length; curlyDepth -= (value.match(/\}/g) || []).length; diff --git a/src/ext/simple_tokenizer_test.js b/src/ext/simple_tokenizer_test.js index d60183ce16b..6f5a71446ad 100644 --- a/src/ext/simple_tokenizer_test.js +++ b/src/ext/simple_tokenizer_test.js @@ -69,3 +69,7 @@ module.exports = { } }; + +if (typeof module !== "undefined" && module === require.main) { + require("asyncjs").test.testcase(module.exports).exec(); +} diff --git a/src/layer/text_util.js b/src/layer/text_util.js index 97d197db877..03df39b4570 100644 --- a/src/layer/text_util.js +++ b/src/layer/text_util.js @@ -2,5 +2,5 @@ const textTokens = new Set(["text", "rparen", "lparen"]); exports.isTextToken = function(tokenType) { - return textTokens.has(tokenType); + return textTokens.has(tokenType.toString()); }; diff --git a/src/mode/_test/highlight_rules_test.js b/src/mode/_test/highlight_rules_test.js index e0db6898877..ceabdf66009 100755 --- a/src/mode/_test/highlight_rules_test.js +++ b/src/mode/_test/highlight_rules_test.js @@ -102,9 +102,9 @@ function checkModes() { var str = blockComment.start + " " + blockComment.end; str = blockComment.start + str; if (blockComment.nestable) - str += blockComment.end; + str += blockComment.end; var data = tokenizer.getLineTokens(str, "start"); - var isBroken = data.tokens.some(function(t) { return !/comment/.test(t.type); }); + var isBroken = data.tokens.some(function(t) { return !/comment|empty/.test(t.type); }); if (isBroken) { die("broken blockComment in " + modeName, data); } @@ -222,7 +222,9 @@ function generateTestData(names, force) { var tokenizedLine = ""; data.tokens.forEach(function(x) { tokenizedLine += x.value; - tmp.push(JSON.stringify([x.type, x.value])); + if (x.type != "empty") { + tmp.push(JSON.stringify([x.type, x.value])); + } }); if (tokenizedLine != line) tmp.push(JSON.stringify(line)); @@ -276,6 +278,7 @@ function testMode(modeName, i) { line = lineData.values.join(""); var tokens = tokenizer.getLineTokens(line, state); + tokens.tokens = tokens.tokens.filter((el) => el.type != "empty"); var values = tokens.tokens.map(function(x) {return x.value;}); var types = tokens.tokens.map(function(x) {return x.type;}); diff --git a/src/mode/folding/mixed.js b/src/mode/folding/mixed.js index 996584a62d5..683d65e1004 100644 --- a/src/mode/folding/mixed.js +++ b/src/mode/folding/mixed.js @@ -13,7 +13,7 @@ oop.inherits(FoldMode, BaseFoldMode); this.$getMode = function(state) { - if (typeof state != "string") + if (Array.isArray(state)) state = state[0]; for (var key in this.subModes) { if (state.indexOf(key) === 0) diff --git a/src/mode/lua_highlight_rules.js b/src/mode/lua_highlight_rules.js index b03286dc78b..eba8b29a61b 100644 --- a/src/mode/lua_highlight_rules.js +++ b/src/mode/lua_highlight_rules.js @@ -7,13 +7,13 @@ var LuaHighlightRules = function() { var keywords = ( "break|do|else|elseif|end|for|function|if|in|local|repeat|"+ - "return|then|until|while|or|and|not" + "return|then|until|while|or|and|not" ); var builtinConstants = ("true|false|nil|_G|_VERSION"); var functions = ( - // builtinFunctions + // builtinFunctions "string|xpcall|package|tostring|print|os|unpack|require|"+ "getfenv|setmetatable|next|assert|tonumber|io|rawequal|"+ "collectgarbage|getmetatable|module|rawset|math|debug|"+ @@ -34,9 +34,9 @@ var LuaHighlightRules = function() { "setupvalue|getlocal|getregistry|getfenv|setn|insert|getn|"+ "foreachi|maxn|foreach|concat|sort|remove|resume|yield|"+ "status|wrap|create|running|"+ - // metatableMethods + // metatableMethods "__add|__sub|__mod|__unm|__concat|__lt|__index|__call|__gc|__metatable|"+ - "__mul|__div|__pow|__len|__eq|__le|__newindex|__tostring|__mode|__tonumber" + "__mul|__div|__pow|__len|__eq|__le|__newindex|__tostring|__mode|__tonumber" ); var stdLibaries = ("string|package|os|io|math|debug|table|coroutine"); @@ -64,27 +64,22 @@ var LuaHighlightRules = function() { this.$rules = { "start" : [{ stateName: "bracketedComment", - onMatch : function(value, currentState, stack){ - stack.unshift(this.next, value.length - 2, currentState); - return "comment"; + onMatch2 : function(value, scope){ + return scope.get(this.next, value.length - 2).get("comment"); }, regex : /\-\-\[=*\[/, next : [ { - onMatch : function(value, currentState, stack) { - if (value.length == stack[1]) { - stack.shift(); - stack.shift(); - this.next = stack.shift(); + onMatch2 : function(value, scope) { + if (scope == "bracketedComment" && value.length == scope.data) { + return scope.parent.get("comment"); } else { - this.next = ""; + return scope.get("comment"); } - return "comment"; }, regex : /\]=*\]/, - next : "start" }, { - defaultToken: "comment.body" + defaultToken : "comment.body" } ] }, @@ -95,26 +90,21 @@ var LuaHighlightRules = function() { }, { stateName: "bracketedString", - onMatch : function(value, currentState, stack){ - stack.unshift(this.next, value.length, currentState); - return "string.start"; + onMatch2 : function(value, scope){ + return scope.get(this.next, value.length).get("string.start"); }, regex : /\[=*\[/, next : [ { - onMatch : function(value, currentState, stack) { - if (value.length == stack[1]) { - stack.shift(); - stack.shift(); - this.next = stack.shift(); + onMatch2 : function(value, scope) { + if (scope == "bracketedString" && value.length == scope.data) { + return scope.parent.get("string.end"); } else { - this.next = ""; + return scope.get("string.end"); } - return "string.end"; }, - + regex : /\]=*\]/, - next : "start" }, { defaultToken : "string" } @@ -149,7 +139,7 @@ var LuaHighlightRules = function() { regex : "\\s+|\\w+" } ] }; - + this.normalizeRules(); }; diff --git a/src/mode/text.js b/src/mode/text.js index 7bdc201a8ee..f775e897c18 100644 --- a/src/mode/text.js +++ b/src/mode/text.js @@ -32,7 +32,14 @@ Mode = function() { this.getTokenizer = function() { if (!this.$tokenizer) { this.$highlightRules = this.$highlightRules || new this.HighlightRules(this.$highlightRuleConfig); - this.$tokenizer = new Tokenizer(this.$highlightRules.getRules()); + var modeName; + if (this.$id) { + modeName = this.$id.split('/').pop(); + } + else { + modeName = "root"; + } + this.$tokenizer = new Tokenizer(this.$highlightRules.getRules(), modeName); } return this.$tokenizer; }; diff --git a/src/mode/text_highlight_rules.js b/src/mode/text_highlight_rules.js index a866846445e..4b99d3679d1 100644 --- a/src/mode/text_highlight_rules.js +++ b/src/mode/text_highlight_rules.js @@ -99,12 +99,16 @@ TextHighlightRules = function() { }; var pushState = function(currentState, stack) { - if (currentState != "start" || stack.length) - stack.unshift(this.nextState, currentState); + if (typeof currentState.get == "function") { + return currentState.get(this.nextState); + } + if (currentState !== "start" || stack.length) stack.unshift(this.nextState, currentState); return this.nextState; }; var popState = function(currentState, stack) { - // if (stack[0] === currentState) + if (typeof currentState.get == "function") { + return (currentState.parent.parent) ? currentState.parent : currentState.get("start"); + } stack.shift(); return stack.shift() || "start"; }; diff --git a/src/scope.js b/src/scope.js new file mode 100644 index 00000000000..eef526d0739 --- /dev/null +++ b/src/scope.js @@ -0,0 +1,132 @@ +"use strict"; + +class Scope { + /** + * @param {any} name + * @param {Scope} [parent] + */ + constructor(name, parent) { + /**@type {Scope & String}*/ + // @ts-ignore + // eslint-disable-next-line no-new-wrappers + let stringObj = new String(name); //TODO: can be removed when ace builds switches to classes + stringObj.name = stringObj.toString(); // Assign the string representation to `name` + stringObj.children = {}; // Initialize the children object + stringObj.parent = parent; // Set the parent scope + stringObj.data = name; // Store the original data + + stringObj.get = this.get; + stringObj.find = this.find; + stringObj.hasParent = this.hasParent; + stringObj.count = this.count; + stringObj.getAllScopeNames = this.getAllScopeNames; + stringObj.toStack = this.toStack; + stringObj.fromStack = this.fromStack; + + return stringObj; + } + + /** + * @param {any} name + * @param {string|undefined} [extraId] + */ + get(name, extraId) { + var id = "" + name + (extraId || ""); + if (this.children[id]) { + return this.children[id]; + } + this.children[id] = new Scope(name, this); + if (extraId) { + this.children[id].data = extraId; + } + return this.children[id]; + } + + find(states) { + /**@type{Scope}*/ + var s = this; + while (s && !states[s.name]) { + s = s.parent; + } + return states[s ? s.name : "start"]; + } + + hasParent(states) { + /**@type{Scope}*/ + var s = this; + while (s && states !== s.name) { + s = s.parent; + } + return s != undefined; + } + + count() { + var s = 1; + for (var i in this.children) s += this.children[i].count(); + return s; + } + + /** + * + * @returns {string[]} + */ + getAllScopeNames() { + var scopeNames = []; + /**@type{Scope}*/ + var self = this; + do { + scopeNames.push(self.name); + } while (self = self.parent); + return scopeNames; + } + + toStack() { + var stack = []; + /**@type{Scope}*/ + var self = this; + do { + stack.push(self.data); + } while (self = self.parent); + + stack.pop(); //drop the root scope name + + if (this.data === "#tmp") { + stack.shift(); + } + if (stack.length === 1) { + stack = []; + } + + return stack; + } + + /** + * Retrieves the scope for the given stack and current state. + * + * @param {string[]} stack - The stack of scopes. + * @param {string | Scope} currentState - The current state. + * @returns {Scope} The scope for the given stack and current state. + */ + fromStack(stack, currentState) { + /**@type{Scope}*/ + let scope = this; + while (scope.parent) { + scope = scope.parent; + } + + if (stack.length === 0) { + return scope.get(currentState); //the start state + } + for (var i = stack.length - 1; i >= 0; i--) { + scope = scope.get(stack[i]); + } + if (stack[0] != currentState) { + scope = scope.get(currentState, "#tmp"); + } + + return scope; + } +} + + +exports.Scope = Scope; diff --git a/src/tokenizer.js b/src/tokenizer.js index 900651d7ca3..64f75ec8d8b 100644 --- a/src/tokenizer.js +++ b/src/tokenizer.js @@ -1,6 +1,7 @@ "use strict"; const reportError = require("./lib/report_error").reportError; +var {Scope} = require("./scope"); // tokenizing lines longer than this makes editor very slow var MAX_TOKEN_COUNT = 2000; /** @@ -10,11 +11,13 @@ class Tokenizer { /** * Constructs a new tokenizer based on the given rules and flags. * @param {Object} rules The highlighting rules - **/ - constructor(rules) { + * @param {string} [modeName] + */ + constructor(rules, modeName) { /**@type {RegExp}*/ this.splitRegex; this.states = rules; + this.rootScope = new Scope(modeName || "root"); this.regExps = {}; this.matchMappings = {}; @@ -203,36 +206,32 @@ class Tokenizer { if (lastCapture.end != null && /^\)*$/.test(src.substr(lastCapture.end))) src = src.substring(0, lastCapture.start) + src.substr(lastCapture.end); } - + // this is needed for regexps that can match in multiple ways if (src.charAt(0) != "^") src = "^" + src; if (src.charAt(src.length - 1) != "$") src += "$"; - + return new RegExp(src, (flag||"").replace("g", "")); } /** * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state. - * @param {string} line - * @param {string | string[]} startState - * @returns {{tokens:import("../ace-internal").Ace.Token[], state: string|string[]}} - */ + * @param {String} line + * @param {String|Scope} [startState] + * @returns {Object} + **/ getLineTokens(line, startState) { - if (startState && typeof startState != "string") { - /**@type {any[]}*/ - var stack = startState.slice(0); - startState = stack[0]; - if (startState === "#tmp") { - stack.shift(); - startState = stack.shift(); - } - } else + if (startState && startState["getAllScopeNames"]) { + var stack = /**@type{Scope}*/(startState).toStack(); + } + else { var stack = []; - - var currentState = /**@type{string}*/(startState) || "start"; + } + var currentState = (startState != undefined) ? (startState["getAllScopeNames"]) ? /**@type{Scope}*/(startState) : this.rootScope.get( + startState) : this.rootScope.get("start"); var state = this.states[currentState]; if (!state) { - currentState = "start"; + currentState = this.rootScope.get("start"); state = this.states[currentState]; } var mapping = this.matchMappings[currentState]; @@ -243,47 +242,78 @@ class Tokenizer { var lastIndex = 0; var matchAttempts = 0; - var token = {type: null, value: ""}; + var token = { + type: null, + value: "" + }; while (match = re.exec(line)) { var type = mapping.defaultToken; var rule = null; + var rememberedState = undefined; var value = match[0]; var index = re.lastIndex; if (index - value.length > lastIndex) { var skipped = line.substring(lastIndex, index - value.length); - if (token.type == type) { + if (token.type && token.type == type) { token.value += skipped; - } else { - if (token.type) - tokens.push(token); - token = {type: type, value: skipped}; + } + else { + if (token.type && token.type != "") tokens.push(token); + token = { + type: currentState.get(type), + value: skipped + }; } } - for (var i = 0; i < match.length-2; i++) { - if (match[i + 1] === undefined) - continue; + for (var i = 0; i < match.length - 2; i++) { + if (match[i + 1] === undefined) continue; rule = state[mapping[i]]; - if (rule.onMatch) - type = rule.onMatch(value, currentState, stack, line); - else - type = rule.token; + if (rule.onMatch || rule.onMatch2) { + if (rule.onMatch) { + type = rule.onMatch(value, currentState.toString(), stack, line); + } + else { + type = rule.onMatch2(value, currentState, line); + rememberedState = (Array.isArray(type)) ? type[0].type.parent : type.parent; + } - if (rule.next) { - if (typeof rule.next == "string") { - currentState = rule.next; - } else { - currentState = rule.next(currentState, stack); + } + else { + if (rule.token != undefined) type = rule.token; + } + + if (rule.next || rule.next2 || rememberedState) { + if (!rememberedState) { + if (rule.next && typeof rule.next !== 'function') { + currentState = currentState.parent.get(rule.next); + } + else { + if (rule.next) { + currentState = rule.next(currentState.toString(), stack); + currentState = this.rootScope.fromStack(stack, currentState); + } + else { + currentState = rule.next2(currentState); + + stack = currentState.toStack(); + } + } } - + else { + currentState = rememberedState; + + stack = currentState.toStack(); + } + state = this.states[currentState]; if (!state) { this.reportError("state doesn't exist", currentState); - currentState = "start"; + currentState = this.rootScope.get("start"); state = this.states[currentState]; } mapping = this.matchMappings[currentState]; @@ -291,31 +321,43 @@ class Tokenizer { re = this.regExps[currentState]; re.lastIndex = index; } - if (rule.consumeLineEnd) - lastIndex = index; + if (rule.consumeLineEnd) lastIndex = index; break; } if (value) { - if (typeof type === "string") { - if ((!rule || rule.merge !== false) && token.type === type) { + if (type && !type["getAllScopeNames"]) { + currentState = this.rootScope.fromStack(stack, currentState); + } + + if (type && !Array.isArray(type) && type != "") { + if ((!rule || rule.merge !== false) && token.type == type) { token.value += value; - } else { - if (token.type) - tokens.push(token); - token = {type: type, value: value}; } - } else if (type) { - if (token.type) - tokens.push(token); - token = {type: null, value: ""}; - for (var i = 0; i < type.length; i++) + else { + if (token.type) tokens.push(token); + token = { + type: currentState.get(type), + value: value + }; + } + } + else if (type) { + if (token.type) tokens.push(token); + token = { + type: null, + value: "" + }; + for (var i = 0; i < type.length; i++) { + if (type[i].type) { + type[i].type = currentState.get(type[i].type); + } tokens.push(type[i]); + } } } - if (lastIndex == line.length) - break; + if (lastIndex === line.length) break; lastIndex = index; @@ -328,31 +370,35 @@ class Tokenizer { } // chrome doens't show contents of text nodes with very long text while (lastIndex < line.length) { - if (token.type) - tokens.push(token); + if (token.type) tokens.push(token); token = { value: line.substring(lastIndex, lastIndex += 500), - type: "overflow" + type: currentState.get("overflow") }; } - currentState = "start"; + currentState = this.rootScope.get("start"); stack = []; break; } } - if (token.type) + if (token.type) tokens.push(token); + + if (!tokens.length || tokens[tokens.length - 1].type && tokens[tokens.length - 1].type.parent !== currentState) { + token = { + value: "", + type: currentState.get("empty") + }; tokens.push(token); - - if (stack.length > 1) { - if (stack[0] !== currentState) - stack.unshift("#tmp", currentState); } + return { - tokens : tokens, - state : stack.length ? stack : currentState + tokens: tokens, + state: currentState }; } + + } Tokenizer.prototype.reportError = reportError;