Improve syntax highlighting

Author: can-lehmann

buffer.nim

@@ -158,9 +158,7 @@ proc delete_tokens*(buffer: Buffer, start: int) =
   while buffer.tokens.len > 0 and
         buffer.tokens[buffer.tokens.len - 1].stop >= start:
     tok = buffer.tokens.pop()
-  while not tok.can_stop and
-        tok.state.requires_stop_token() and
-        buffer.tokens.len > 0:
+  while not tok.can_stop and buffer.tokens.len > 0:
     tok = buffer.tokens.pop()
   buffer.tokens_done = false
 

editor.nim

@@ -1488,7 +1488,7 @@ method render(editor: Editor, box: Box, ren: var TermRenderer) =
 
 method close*(editor: Editor) =
   editor.buffer.unregister_hook(editor.cursor_hook_id)  
-  
+
 proc make_editor*(app: App, buffer: Buffer): Editor =
   result = Editor(
     buffer: buffer,

highlight/cpp.nim

@@ -76,7 +76,7 @@ method next(state: State, text: seq[Rune]): Token =
       case chr:
         of '\n':
           let state = State(it: start + 1)
-          return Token(kind: TokenUnknown, start: start, stop: start + 1, state: state)
+          return Token(kind: TokenUnknown, start: start, stop: start + 1, state: state, can_stop: true)
         of '\"':
           let
             it = text.skip_string_like(start + 1, end_char='\"')
@@ -111,12 +111,15 @@ method next(state: State, text: seq[Rune]): Token =
     of ModeNone:
       let start = text.skip_whitespace(state.it)
       if start >= text.len:
-        return Token(kind: TokenNone)  
+        return Token(kind: TokenNone)
       let chr = text[start]
       template skip_chr() =
         let state = State(it: start + 1)
         return Token(kind: TokenUnknown, start: start, stop: start + 1, state: state)
       case chr:
+        of '\n':
+          let state = State(it: start + 1)
+          return Token(kind: TokenUnknown, start: start, stop: start + 1, state: state, can_stop: true)
         of '\"', '\'':
           let
             it = text.skip_string_like(start + 1, end_char=chr)

highlight/css.nim

@@ -71,7 +71,7 @@ method next(state: State, text: seq[Rune]): Token =
 
   template char_token(new_state) =
     return Token(kind: TokenUnknown,
-      start: start, stop: start + 1, state: new_state
+      start: start, stop: start + 1, state: new_state, can_stop: true
     )
 
   template skip_token() =

highlight/highlight.nim

@@ -45,13 +45,12 @@ type
     can_stop*: bool
 
 method next*(state: HighlightState, text: seq[Rune]): Token {.base.} = quit "Abstract"
-method requires_stop_token*(state: HighlightState): bool {.base.} = false
 
 proc is_inside*(token: Token, index: int): bool =
   index >= token.start and index < token.stop
 
 proc is_whitespace*(rune: Rune): bool =
-  rune == ' ' or rune == '\t' or rune == '\r' or rune == '\n'
+  rune == ' ' or rune == '\t' or rune == '\r'
 
 proc skip_whitespace*(text: seq[Rune], pos: int): int =
   result = pos

highlight/html.nim

@@ -34,6 +34,9 @@ method next*(state: State, text: seq[Rune]): Token =
     return Token(kind: TokenNone)  
   let chr = text[start]
   case chr:
+    of '\n':
+      let state = State(it: start + 1, is_tag: state.is_tag, is_close: state.is_close)
+      return Token(kind: TokenUnknown, start: start, stop: start + 1, state: state, can_stop: true)
     of '\"':
       let
         it = text.skip_string_like(start + 1)

highlight/json.nim

@@ -45,9 +45,9 @@ method next(state: State, text: seq[Rune]): Token =
         stop = text.skip_string_like(start + 1, end_char = chr)
         state = State(it: stop + 1)
       return Token(kind: TokenString, start: start, stop: stop + 1, state: state)
-    of ':', ',', '[', ']', '{', '}':
+    of ':', ',', '[', ']', '{', '}', '\n':
       let state = State(it: start + 1)
-      return Token(kind: TokenUnknown, start: start, stop: start + 1, state: state)
+      return Token(kind: TokenUnknown, start: start, stop: start + 1, state: state, can_stop: true)
     else:
       var
         name: seq[Rune] = @[]

highlight/lisp.nim

@@ -57,15 +57,18 @@ proc token_kind(name: seq[Rune], is_call: bool, is_quote: bool): TokenKind =
 method next*(state: State, text: seq[Rune]): Token =
   var start = text.skip_whitespace(state.it)
   if start >= text.len:
-    return Token(kind: TokenNone)  
+    return Token(kind: TokenNone)
   let chr = text[start]
   case chr:
+    of '\n':
+      let new_state = State(it: start + 1, depth: state.depth, quote_depths: state.quote_depths)
+      return Token(kind: TokenUnknown, start: start, stop: start + 1, state: new_state, can_stop: true)
     of ';':
       var it = start + 1
       while it < text.len and text[it] != '\n':
         it += 1
       let new_state = State(it: it + 1, depth: state.depth, quote_depths: state.quote_depths)
-      return Token(kind: TokenComment, start: start, stop: it + 1, state: new_state)
+      return Token(kind: TokenComment, start: start, stop: it + 1, state: new_state, can_stop: true)
     of '\"':
       let
         it = text.skip_string_like(start + 1)

highlight/lua.nim

@@ -74,7 +74,7 @@ method next*(state: State, text: seq[Rune]): Token =
             name.add(chr)
             it += 1
       if name.len == 0:
-        return Token(kind: TokenUnknown, start: start, stop: it + 1, state: State(it: it + 1))
+        return Token(kind: TokenUnknown, start: start, stop: it + 1, state: State(it: it + 1), can_stop: true)
       return Token(kind: name.token_kind, start: start, stop: it, state: State(it: it + 1))
 
 proc new_lua_highlighter*(): HighlightState = State()

highlight/markdown.nim

@@ -38,10 +38,8 @@ proc token_kind(state: State): TokenKind =
     return TokenItalic
   return TokenUnknown
 
-method requires_stop_token(state: State): bool = true
-
 method next(state: State, text: seq[Rune]): Token =
-  let start = text.skip(state.it, {' ', '\r', '\t'})
+  let start = text.skip_whitespace(state.it)
   if start >= text.len:
     return Token(kind: TokenNone)  
   let chr = text[start]

highlight/nim.nim

@@ -26,7 +26,6 @@ import highlight, ../utils
 type
   State = ref object of HighlightState
     it: int
-    is_float: bool
 
 const
   NIM_KEYWORDS = [
@@ -53,7 +52,8 @@ const
     "bool", "char", "auto", "pointer",
     "cint", "cshort", "cstring", "clong",
     "cuint", "culong", "cushort",
-    "openArray", "Table", "Deque", "HashSet", "range"
+    "openArray", "Table", "Deque", "HashSet", "range",
+    "untyped", "typed", "varargs"
   ]
 
 
@@ -64,7 +64,9 @@ proc token_kind(str: seq[Rune]): TokenKind =
     return TokenKeyword
   elif $str in NIM_TYPES:
     return TokenType
-  elif str.is_int(allow_underscore=true) or str.is_float(allow_underscore=true):
+  elif str.is_int(allow_underscore=true) or
+       str.is_float(allow_underscore=true) or
+       ('\'' in str and str.split('\'')[0].is_float(true)):
     return TokenLiteral
 
   return TokenName
@@ -80,11 +82,16 @@ method next*(state: State, text: seq[Rune]): Token =
         it = text.skip_string_like(start + 1)
         state = State(it: it + 1)
       return Token(kind: TokenString, start: start, stop: it + 1, state: state)
-    of '\'':  
+    of '\'':
       let
         it = text.skip_string_like(start + 1, end_char='\'')
         state = State(it: it + 1)
       return Token(kind: TokenChar, start: start, stop: it + 1, state: state)
+    of '`':
+      let
+        it = text.skip_string_like(start + 1, end_char='`')
+        state = State(it: it + 1)
+      return Token(kind: TokenName, start: start, stop: it + 1, state: state)
     of '#':
       if start + 1 < text.len and text[start + 1] == '[':
         var
@@ -106,37 +113,39 @@ method next*(state: State, text: seq[Rune]): Token =
           it += 1
         let state = State(it: it + 1)
         return Token(kind: TokenComment, start: start, stop: it, state: state)
-    of ':', '<', '>', '[', ']', '(', ')', '{', '}', ',', ';', '=', '`':
+    of ':', '.', '[', ']', '(', ')', '{', '}', ',', ';':
       let state = State(it: start + 1)
       return Token(kind: TokenUnknown, start: start, stop: start + 1, state: state)
-    of '.':
-      var
-        kind = TokenUnknown
-        offset = 1
-      if state.is_float and
-         start + 1 < text.len and
-         is_digit(text[start + 1]):
-        kind = TokenLiteral
-        offset = 2
+    of '\n':
       let state = State(it: start + 1)
-      return Token(kind: kind, start: start, stop: start + offset, state: state)
+      return Token(kind: TokenUnknown, start: start, stop: start + 1, state: state, can_stop: true)
     else:
+      type NameKind = enum NameNone, NameOperator, NameText
+      
       var
         name: seq[Rune] = @[]
+        kind = NameNone
         it = start
       while it < text.len:
         let chr = text[it]
         case chr:
-          of ' ', '\t', '\n', '\r', ':', '<', '>',
+          of ' ', '\t', '\n', '\r', ':',
              '[', ']', '(', ')', '{', '}', ',', ';',
-             '=', '`', '\"', '#', '\'', '.':
-            break
+             '`', '\"', '#', '\'', '.':
+            if not (name.is_float(true) and chr == '\''):
+              break
+          of '+', '-', '*', '/', '=', '<', '>', '!', '$', '%', '&', '|', '?':
+            if kind == NameText:
+              break
+            kind = NameOperator
           else:
-            name.add(chr)
+            if kind == NameOperator and
+               not (name.is_int(true) and chr == '.'):
+              break
+            kind = NameText
+        name.add(chr)
         it += 1
-      let
-        kind = name.token_kind()
-        state = State(it: it, is_float: name.is_float(allow_underscore=true))
-      return Token(kind: kind, start: start, stop: it, state: state)
+      let state = State(it: it)
+      return Token(kind: name.token_kind(), start: start, stop: it, state: state)
 
 proc new_nim_highlighter*(): HighlightState = State()