Fix PythonPlugin builtin module discovery

Author: barnabasdomozi

plugins/python/parser/pyparser/parserconfig.py

@@ -8,7 +8,7 @@ class ParserConfig:
     sys_path: List[str]
     venv_path: str | None = None
     project: Project | None = None 
-    debug: bool = False
+    debug: bool = True
     stack_trace: bool = False
     safe_env: bool = False
     type_hint: bool = False

plugins/python/parser/pyparser/pybuiltin.py

@@ -2,34 +2,44 @@
 import os
 import importlib.util
 import traceback
+import sysconfig
 from jedi.api.classes import Name
 from parserlog import log, bcolors
 from parserconfig import ParserConfig
 
 class PYBuiltin:
     builtin = {}
 
+    @staticmethod
+    def __searchDirectory(directory: str):
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                p = os.path.join(root, file)
+                ext = os.path.splitext(p)[1]
+
+                if ext and ext.lower() == '.py':
+                    PYBuiltin.builtin[p] = True
+
     @staticmethod
     def findBuiltins(config: ParserConfig):
         try:
+            # Consider all Python modules in the stdlib directory builtin
+            sysconfig_paths = sysconfig.get_paths()
+            if "stdlib" in sysconfig_paths:
+                PYBuiltin.__searchDirectory(sysconfig_paths["stdlib"])
+
+            # Locate module paths via ModuleSpec
+            # However, this can return "frozen" and "built-in" as module origin and not the actual file
             # Note: Python 3.10+ required
             stdlib_modules = sys.stdlib_module_names
-
             for e in stdlib_modules:
                 spec = importlib.util.find_spec(e)
-                if spec and spec.origin:
+                if spec and spec.origin and spec.origin != "frozen" and spec.origin != "built-in":
                     PYBuiltin.builtin[spec.origin] = True
 
                 if spec and spec.submodule_search_locations:
                     for submodule_dir in spec.submodule_search_locations:
-                        for root, dirs, files in os.walk(submodule_dir):
-                            for file in files:
-                                p = os.path.join(root, file)
-                                ext = os.path.splitext(p)[1]
-
-                                if ext and ext.lower() == '.py':
-                                    PYBuiltin.builtin[p] = True
-
+                        PYBuiltin.__searchDirectory(submodule_dir)
         except:
             log(f"{bcolors.FAIL}Failed to find Python builtins!")
             if config.stack_trace:

plugins/python/parser/pyparser/pyname.py

@@ -98,6 +98,7 @@ def getNodeInfo(self) -> NodeInfo:
         node.is_definition = self.name.is_definition()
         node.file_id = self.__getFileId()
         node.type_hint = self.__getNameTypeHint()
+        node.type_hint = str(self.path) + ":" + str(list(map(lambda x : x.path, self.defs)))
         node.is_builtin = PYBuiltin.isBuiltin(self.name) or any(list(map(lambda x : PYBuiltin.isBuiltin(x.name), self.defs)))
 
         parent = self.name.parent()

plugins/python/test/src/pythonparsertest.cpp

@@ -26,15 +26,14 @@ class PythonParserTest : public ::testing::Test
 
   model::PYName queryFile(const std::string& filename, const odb::query<model::PYName>& odb_query)
   {
-    model::PYName pyname;
-    if (m_files.count(filename))
+    if(!m_files.count(filename))
     {
-      _transaction([&, this]() {
-        pyname = _db->query_value<model::PYName>(odb_query && odb::query<model::PYName>::file_id == m_files[filename]);
-      });
+      throw std::runtime_error("File " + filename + " not found!");
     }
 
-    return pyname;
+    return _transaction([&, this]() {
+      return _db->query_value<model::PYName>(odb_query && odb::query<model::PYName>::file_id == m_files[filename]);
+    });
   }
 
 private:
@@ -422,6 +421,8 @@ TEST_F(PythonParserTest, BuiltinVariable)
         (odb::query<model::PYName>::line_start == 6 &&
         odb::query<model::PYName>::value == "print"));
 
+  LOG(info) << pyname.type_hint;
+
   EXPECT_EQ(pyname.is_builtin, true);
 
   pyname = queryFile("imports.py",