implement a json5 lexer (#1561)

* implement a json5 lexer

* add a desc

* Remove unnecessary rules

* Mirror pattern from Pygments JSON5

* Document JSON5 support

---------

Co-authored-by: http://jneen.net/ <[email protected]>
Co-authored-by: Michael Camilleri <[email protected]>
Co-authored-by: Tan Le <[email protected]>

docs/Languages.md

@@ -103,6 +103,7 @@
 - Jinja (`jinja`)
 - JSL (`jsl`)
 - JSON (`json`)
+- JSON5 (`json5`)
 - Json-doc (`json-doc`)
 - Jsonnet (`jsonnet`)
 - Jsp (`jsp`)

lib/rouge/demos/json5

@@ -0,0 +1,12 @@
+{
+  // comments
+  unquoted: 'and you can quote me on that',
+  singleQuotes: 'I can use "double quotes" here',
+  lineBreaks: "Look, Mom! \
+No \\n's!",
+  hexadecimal: 0xdecaf,
+  leadingDecimalPoint: .8675309, andTrailing: 8675309.,
+  positiveSign: +1,
+  trailingComma: 'in objects', andIn: ['arrays',],
+  "backwardsCompatible": "with JSON",
+}

lib/rouge/lexers/json5.rb

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'javascript.rb'
+
+    class JSON5 < JSON
+      title 'JSON 5'
+      tag 'json5'
+      filenames '*.json5'
+      mimetypes 'application/json5', 'application/x-json5'
+
+      desc 'JSON 5 extension for JSON (json5.org)'
+
+      append :whitespace do
+        rule %r://.*$:, Comment
+
+        # comments are non-nesting, so a single regex will do
+        rule %r:/[*].*?[*]/:m, Comment
+      end
+
+      prepend :name do
+        rule Javascript.id_regex, Name::Label
+        rule %r/".*?"/, Name::Label
+      end
+
+      # comments can appear between keys and :, so we have to
+      # manage our states a little more carefully
+      append :object do
+        rule %r/:/ do
+          token Punctuation
+          goto :object_value
+        end
+      end
+
+      state :object_value do
+        mixin :value
+        rule %r/,/ do
+          token Punctuation
+          goto :object
+        end
+
+        rule %r/}/, Punctuation, :pop!
+      end
+
+      append :value do
+        rule %r/'/, Str::Single, :sstring
+      end
+
+      state :sstring do
+        rule %r/[^\\']+/, Str::Single
+        rule %r/\\./m, Str::Escape
+        rule %r/'/, Str::Single, :pop!
+      end
+
+      # can escape newlines
+      append :string do
+        rule %r/\\./m, Str::Escape
+      end
+
+      # override: numbers are very different in json5
+      state :constants do
+        rule %r/\b(?:true|false|null)\b/, Keyword::Constant
+        rule %r/[+-]?\b(?:Infinity|NaN)\b/, Keyword::Constant
+        rule %r/[+-]?0x\h+/i, Num::Hex
+
+        rule %r/[+-.]?[0-9]+[.]?[0-9]?([eE][-]?[0-9]+)?/i, Num::Float
+        rule %r/[+-]?\d+e[+-]?\d+/, Num::Integer
+        rule %r/[+-]?(?:0|[1-9]\d*)(?:e[+-]?\d+)?/i, Num::Integer
+      end
+    end
+  end
+end

spec/lexers/json5_spec.rb

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+describe Rouge::Lexers::JSON5 do
+  let(:subject) { Rouge::Lexers::JSON5.new }
+
+  describe 'guessing' do
+    include Support::Guessing
+
+    it 'guesses by filename' do
+      assert_guess :filename => 'foo.json5'
+    end
+
+    it 'guesses by mimetype' do
+      assert_guess :mimetype => 'application/json5'
+    end
+  end
+end

spec/visual/samples/json5

@@ -0,0 +1,44 @@
+{
+  // comments
+  unquoted: 'and you can quote me on that',
+  singleQuotes: 'I can use "double quotes" here',
+  lineBreaks: "Look, Mom! \
+No \\n's!",
+  hexadecimal: 0xdecaf,
+  leadingDecimalPoint: .8675309, andTrailing: 8675309.,
+  positiveSign: +1,
+  trailingComma: 'in objects', andIn: ['arrays',],
+  "backwardsCompatible": "with JSON",
+
+
+  a: 1e5,
+  b: 1.e-5,
+  c: .1e5,
+
+  /**
+   * multiline
+   * comments
+   */
+
+  d
+  : 3,
+
+  "e"
+  // comment
+  : 4,
+
+  foo: "bar",
+  "bar": "foo",
+
+  f //comment
+  : 5,
+
+  constants: [
+    NaN, Infinity, true, false, null,
+    { NaN: NaN,
+      Infinity: Infinity,
+      true: true,
+      false: false,
+      null: null },
+  ]
+}