Merge pull request #37 from Laserlicht/master

more robust json parsing

Author: dydzio0614

requirements.txt

@@ -4,6 +4,7 @@ mkdocs-material==9.5.21
 mkdocs-glightbox==0.3.7 # newer versions causes problem (lightbox works only for cached images)
 mkdocs-rss-plugin==1.12.2
 mdutils==1.6.0
+json_repair==0.20.1
 json5==0.9.25
 pillow==10.3.0
 numpy==1.26.4

src/build.py

@@ -1,10 +1,11 @@
-import json5
 import urllib.request
 import shutil
 from mdutils.mdutils import MdUtils
 import logging
 import sys
 
+from helper import load_vcmi_json
+
 from build_extract_main_repo import build_extract_main_repo
 from build_mod_overview import build_mod_overview
 from build_mod_page import build_mod_page
@@ -20,13 +21,13 @@
 shutil.rmtree("docs", ignore_errors=True)
 
 vcmi_url = "https://github.com/vcmi/vcmi/archive/refs/heads/develop.zip"
-settings_schema = json5.loads(urllib.request.urlopen("https://raw.githubusercontent.com/vcmi/vcmi/develop/config/schemas/settings.json").read())
+settings_schema = load_vcmi_json(urllib.request.urlopen("https://raw.githubusercontent.com/vcmi/vcmi/develop/config/schemas/settings.json").read().decode())
 vcmi_mod_url = settings_schema["properties"]["launcher"]["properties"]["defaultRepositoryURL"]["default"]
 #vcmi_mod_url = "https://pastebin.com/raw/MUYS7dbJ" #test
 
 log.info('Download mod repo')
 repo = urllib.request.urlopen(vcmi_url).read()
-repo_mod = json5.loads(urllib.request.urlopen(vcmi_mod_url).read())
+repo_mod = load_vcmi_json(urllib.request.urlopen(vcmi_mod_url).read().decode())
 
 log.info('Create main page')
 build_extract_main_repo(repo)

src/build_mod_overview.py

@@ -1,16 +1,17 @@
 import os
-import json5
 import urllib.request
 from mdutils.mdutils import MdUtils
 from mdutils.tools import Html
 from mdutils.tools.Link import Inline
 
+from helper import load_vcmi_json
+
 def build_mod_overview(mod_repo, cb):
     os.makedirs("docs/Mod Repository", exist_ok=True)
     mdModOverview = MdUtils(file_name='docs/Mod Repository/Overview', title='Overview')
     mdModOverviewTable = ["Mod", "Type", "Description", "Version", "Translations"]
     for key, value in mod_repo.items():
-        mod = json5.loads(urllib.request.urlopen(value["mod"].replace(" ", "%20")).read())
+        mod = load_vcmi_json(urllib.request.urlopen(value["mod"].replace(" ", "%20")).read().decode())
 
         translations = [k for k, v in mod.items() if isinstance(v, dict) and "translations" in v]
 

src/helper.py

@@ -1,2 +1,30 @@
+import re
+import json_repair
+import json5
+
 def get_value_if_exists(elem, val):
-    return elem[val] if val in elem else ""
+    return elem[val] if val in elem else ""
+
+# https://stackoverflow.com/a/18381470 (Onur Yıldırım, CC BY-SA 4.0)
+def remove_comments(string):
+    pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)"
+    # first group captures quoted strings (double or single)
+    # second group captures comments (//single-line or /* multi-line */)
+    regex = re.compile(pattern, re.MULTILINE|re.DOTALL)
+    def _replacer(match):
+        # if the 2nd group (capturing comments) is not None,
+        # it means we have captured a non-quoted (real) comment string.
+        if match.group(2) is not None:
+            return "" # so we will return empty to remove the comment
+        else: # otherwise, we will return the 1st group
+            return match.group(1) # captured quoted-string
+    return regex.sub(_replacer, string)
+
+def load_vcmi_json(string):
+    try:
+        obj = json5.loads(string)
+    except:
+        tmp = remove_comments(string)
+        obj = json_repair.loads(tmp)
+
+    return obj

src/parse_mod.py

@@ -1,7 +1,6 @@
 import collections.abc
 import io
 import os
-import json5
 import tempfile
 import zipfile
 import urllib.request
@@ -12,6 +11,8 @@
 import logging
 log = logging.getLogger('LOGGER_NAME')
 
+from helper import load_vcmi_json
+
 from defextract import extract_def
 
 def nested_update(d, u):
@@ -53,7 +54,7 @@ def __parse_subconfig(self, data, dir):
                         for file in files:
                             if fullpath.lower() == os.path.join(subdir, file).lower():
                                 log.info('open json: ' + os.path.join(subdir, file))
-                                tmp2 = nested_update(tmp2, json5.load(open(os.path.join(subdir, file))))
+                                tmp2 = nested_update(tmp2, load_vcmi_json(open(os.path.join(subdir, file)).read()))
                 if len(tmp2) > 0:
                     tmp[key.lower()] = tmp2
         return tmp
@@ -64,7 +65,7 @@ def get_mods(self):
             for file in files:
                 if file.lower() == "mod.json":
                     log.info('open json: ' + os.path.join(subdir, file))
-                    data = json5.load(open(os.path.join(subdir, file)))
+                    data = load_vcmi_json(open(os.path.join(subdir, file)).read())
                     mods.append(
                         {
                             "pyhsicaldir": subdir,
@@ -119,7 +120,7 @@ def get_animations(self, mod, path):
                         if file.lower().endswith(".json") or file.lower().endswith(".def"):
                             if file.lower().endswith(".json"):
                                 log.info('open json: ' + os.path.join(subdir, file))
-                                tmp = json5.load(open(os.path.join(subdir, file)))
+                                tmp = load_vcmi_json(open(os.path.join(subdir, file)).read())
                                 for i, sequence in enumerate(tmp["sequences"]):
                                     for j, frame in enumerate(sequence["frames"]):
                                         path_img = tmp["basepath"] + frame