WIP: Fix various issues with config file loading, parsing & generation

mkdoxy/doxyrun.py

@@ -103,16 +103,55 @@ def setDoxyCfg(self, doxyCfgNew: dict) -> dict:
                 "GENERATE_XML": "YES",
                 "RECURSIVE": "YES",
                 "EXAMPLE_PATH": "examples",
-                "SHOW_NAMESPACES": "YES",
-                "GENERATE_HTML": "NO",
-                "GENERATE_LATEX": "NO",
             }
 
-        doxyCfg.update(doxyCfgNew)
-        doxyCfg["INPUT"] = self.doxygenSource
+        # MkDoxy only cares about the XML output. Always override these.
+        overrides = {
+            "GENERATE_XML": "YES",
+            "GENERATE_HTML": "NO",
+            "GENERATE_LATEX": "NO",
+        }
+
+        doxyCfg.update(overrides)
+        doxyCfg["INPUT"] = self.merge_doxygen_input(doxyCfg)
         doxyCfg["OUTPUT_DIRECTORY"] = self.tempDoxyFolder
+        doxyCfg.update(doxyCfgNew)
+
+        if self.doxygenSource and self.doxyConfigFile:
+            log.info(f"Merged `src-dirs` and `INPUT` from `doxy-cfg-file`:\n  INPUT = {doxyCfg['INPUT']}")
+
         return doxyCfg
 
+    def merge_doxygen_input(self, doxyCfg):
+        """! Merge `src-dirs` (if any) with the "INPUT" paths from the `doxy-cfg-file` (if any). Paths are de-duplicated.
+        @details
+        @param doxyCfg: (dict) the current doxygen configuration to merge with.
+        @return: (str) A string containing the relative paths to be set as "INPUT", separated by " ".
+        """
+        doxycfg_input = doxyCfg.get("INPUT", "")
+
+        if not self.doxygenSource or self.doxygenSource == "":
+            return doxycfg_input
+
+        if not doxycfg_input or doxycfg_input == "":
+            return self.doxygenSource
+
+        # `src-dirs` is always relative to the directory containing the `doxy-cfg-file`.
+        abs_run_dir = self.getDoxygenRunFolder().resolve()
+
+        # Make all paths absolute and deduplicate them by pushing into a dictionary.
+
+        # First paths from `src-dirs`. They are relative to the current working directory.
+        abs_path_dict = dict.fromkeys(Path(src_dir).resolve() for src_dir in self.doxygenSource.split(" "))
+        # Now paths from the config file. They are relative to `abs_run_dir`
+        abs_path_dict |= dict.fromkeys(
+            Path.joinpath(abs_run_dir, input_item).resolve() for input_item in doxycfg_input.split(" ")
+        )
+
+        return " ".join(os.path.relpath(abs_path, abs_run_dir) for abs_path in abs_path_dict.keys())
+
+
+
     def is_doxygen_valid_path(self, doxygen_bin_path: str) -> bool:
         """! Check if the Doxygen binary path is valid.
         @details Accepts a full path or just 'doxygen' if it exists in the system's PATH.
@@ -205,7 +244,7 @@ def hashRead(filename: PurePath) -> str:
                 return str(file.read())
 
         sha1 = hashlib.sha1()
-        srcs = self.doxygenSource.split(" ")
+        srcs = self.doxyCfg["INPUT"].split(" ")
         for src in srcs:
             for path in Path(src).rglob("*.*"):
                 # # Code from https://stackoverflow.com/a/22058673/15411117
@@ -238,6 +277,7 @@ def run(self):
             stdout=PIPE,
             stdin=PIPE,
             stderr=PIPE,
+            cwd=self.getDoxygenRunFolder(),
         )
         (doxyBuilder.communicate(self.dox_dict2str(self.doxyCfg).encode("utf-8"))[0].decode().strip())
         # log.info(self.destinationDir)
@@ -261,6 +301,17 @@ def getOutputFolder(self) -> PurePath:
         """
         return Path.joinpath(Path(self.tempDoxyFolder), Path("xml"))
 
+    def getDoxygenRunFolder(self):
+        """! Get the working directory to execute Doxygen in. Important to resolve releative paths.
+        @details When a doxygen config file is provided, this is its containing directory. Otherwise it's the current
+          working directory.
+        @return: (Path) Path to the folder to execute Doxygen in.
+        """
+        if not self.doxyConfigFile:
+            return Path.cwd()
+
+        return Path(self.doxyConfigFile).parent
+
 
 # not valid path exception
 class DoxygenBinPathNotValid(Exception):

mkdoxy/generatorAuto.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from pathlib import Path
 
 from mkdocs.structure import files
 
@@ -61,7 +62,10 @@ def __init__(
     def save(self, path: str, output: str):
         pathRel = os.path.join(self.apiPath, path)
         self.fullDocFiles.append(files.File(pathRel, self.tempDoxyDir, self.siteDir, self.useDirectoryUrls))
-        with open(os.path.join(self.tempDoxyDir, pathRel), "w", encoding="utf-8") as file:
+
+        fullpath = Path(os.path.join(self.tempDoxyDir, pathRel))
+        fullpath.parent.mkdir(parents=True, exist_ok=True)
+        with open(fullpath, "w", encoding="utf-8") as file:
             file.write(output)
 
     def fullDoc(self, defaultTemplateConfig: dict):

mkdoxy/node.py

@@ -41,7 +41,17 @@ def __init__(
             if self.debug:
                 log.info(f"Loading XML from: {xml_file}")
             self._dirname = os.path.dirname(xml_file)
-            self._xml = ElementTree.parse(xml_file).getroot().find("compounddef")
+            try:
+                self._xml = ElementTree.parse(xml_file).getroot().find("compounddef")
+            # XML may contain invalid characters. Attempt to replace these with valid UTF-8 and try again.
+            except ElementTree.ParseError:
+                print(parent)
+                print("  " + xml_file)
+                with open(xml_file, "rb") as file:
+                    contents = file.read().decode("utf-8", "replace")
+                    # ElementTree.fromstring() gets the root element.
+                    self._xml = ElementTree.fromstring(contents).find("compounddef")
+
             if self._xml is None:
                 raise Exception(f"File {xml_file} has no <compounddef>")
             self._kind = Kind.from_str(self._xml.get("kind"))
@@ -107,13 +117,8 @@ def _check_for_children(self):
                     continue
                 except Exception:
                     pass
-            child = Node(
-                os.path.join(self._dirname, f"{refid}.xml"),
-                None,
-                self.project,
-                self._parser,
-                self,
-            )
+
+            child = self._parse_inner_into_node(refid, innergroup.text)
             child._visibility = Visibility.PUBLIC
             self.add_child(child)
 
@@ -131,24 +136,7 @@ def _check_for_children(self):
                 except Exception:
                     pass
 
-            try:
-                child = Node(
-                    os.path.join(self._dirname, f"{refid}.xml"),
-                    None,
-                    self.project,
-                    self._parser,
-                    self,
-                )
-            except FileNotFoundError:
-                child = Node(
-                    os.path.join(self._dirname, f"{refid}.xml"),
-                    Element("compounddef"),
-                    self.project,
-                    self._parser,
-                    self,
-                    refid=refid,
-                )
-                child._name = innerclass.text
+            child = self._parse_inner_into_node(refid, innerclass.text)
             child._visibility = prot
             self.add_child(child)
 
@@ -162,13 +150,7 @@ def _check_for_children(self):
                 except Exception:
                     pass
 
-            child = Node(
-                os.path.join(self._dirname, f"{refid}.xml"),
-                None,
-                self.project,
-                self._parser,
-                self,
-            )
+            child = self._parse_inner_into_node(refid, innerfile.text)
             child._visibility = Visibility.PUBLIC
             self.add_child(child)
 
@@ -203,13 +185,7 @@ def _check_for_children(self):
                 except Exception:
                     pass
 
-            child = Node(
-                os.path.join(self._dirname, f"{refid}.xml"),
-                None,
-                self.project,
-                self._parser,
-                self,
-            )
+            child = self._parse_inner_into_node(refid, innernamespace.text)
             child._visibility = Visibility.PUBLIC
             self.add_child(child)
 
@@ -240,6 +216,27 @@ def _check_for_children(self):
         # if para.find('programlisting') is not None:
         # 	self._programlisting = Property.Programlisting(para, self._parser, self._kind)
 
+    def _parse_inner_into_node(self, refid, fallback_name):
+        xml_path = os.path.join(self._dirname, f"{refid}.xml")
+
+        # Not every item inside <compounddef> has a corresponding XML file on disk (it may not be documented). Check
+        # if it does, otherwise, just create leaf "compounddef" element.
+
+        if os.path.exists(xml_path):
+            child = Node(xml_path, None, self.project, self._parser, self)
+        else:
+            child = Node(
+                xml_path,
+                Element("compounddef"),
+                self.project,
+                self._parser,
+                self,
+                refid=refid,
+            )
+            child._name = fallback_name
+
+        return child
+
     def _check_attrs(self):
         prot = self._xml.get("prot")
         self._visibility = Visibility(prot) if prot is not None else Visibility.PUBLIC