Improve performance by caching find_spec

Certain checkers upstream on pylint like import-error heavily use
find_spec. This method is IO intensive as it looks for files
across several search paths to return a ModuleSpec.

Since imports across files may repeat themselves it makes sense to cache
this method in order to speed up the linting process.

Local testing shows that caching reduces the total amount of calls to
find_module methods (used by find_spec) by about 50%. Linting the test
repository in the related issue goes from 40 seconds to 37 seconds. This
was on a NVME disk and after warmup, so timing gains may be bigger on
slower file systems like the one mentioned in the referenced issue.

Closes pylint-dev/pylint#9310.

Author: crazybolillo

astroid/interpreter/_import/spec.py

@@ -24,6 +24,12 @@
 
 from . import util
 
+_spec_cache = {}
+
+
+def clear_spec_cache():
+    _spec_cache.clear()
+
 
 # The MetaPathFinder protocol comes from typeshed, which says:
 # Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder`
@@ -423,6 +429,18 @@ def _find_spec_with_path(
     raise ImportError(f"No module named {'.'.join(module_parts)}")
 
 
+def spec_cache(func):
+    def wrapper(*args):
+        key = ".".join(args[0])
+        if key not in _spec_cache:
+            _spec_cache[key] = func(*args)
+
+        return _spec_cache[key]
+
+    return wrapper
+
+
+@spec_cache
 def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSpec:
     """Find a spec for the given module.
 

astroid/manager.py

@@ -442,10 +442,12 @@ def clear_cache(self) -> None:
         # pylint: disable=import-outside-toplevel
         from astroid.brain.helpers import register_all_brains
         from astroid.inference_tip import clear_inference_tip_cache
+        from astroid.interpreter._import.spec import clear_spec_cache
         from astroid.interpreter.objectmodel import ObjectModel
         from astroid.nodes._base_nodes import LookupMixIn
         from astroid.nodes.scoped_nodes import ClassDef
 
+        clear_spec_cache()
         clear_inference_tip_cache()
         _invalidate_cache()  # inference context cache
 

tests/test_manager.py

@@ -23,6 +23,7 @@
     AttributeInferenceError,
 )
 from astroid.interpreter._import import util
+from astroid.interpreter._import.spec import clear_spec_cache
 from astroid.modutils import EXT_LIB_DIRS, module_in_path
 from astroid.nodes import Const
 from astroid.nodes.scoped_nodes import ClassDef, Module
@@ -41,6 +42,7 @@ class AstroidManagerTest(
 ):
     def setUp(self) -> None:
         super().setUp()
+        clear_spec_cache()
         self.manager = test_utils.brainless_manager()
 
     def test_ast_from_file(self) -> None:

tests/test_modutils.py

@@ -22,6 +22,7 @@
 from astroid import modutils
 from astroid.const import PY310_PLUS
 from astroid.interpreter._import import spec
+from astroid.interpreter._import.spec import clear_spec_cache
 
 from . import resources
 
@@ -41,6 +42,7 @@ class ModuleFileTest(unittest.TestCase):
     package = "mypypa"
 
     def tearDown(self) -> None:
+        clear_spec_cache()
         for k in list(sys.path_importer_cache):
             if "MyPyPa" in k:
                 del sys.path_importer_cache[k]