Improve performance by caching find_spec

Certain checkers upstream on pylint like import-error heavily use
find_spec. This method is IO intensive as it looks for files
across several search paths to return a ModuleSpec.

Since imports across files may repeat themselves it makes sense to cache
this method in order to speed up the linting process.

Closes pylint-dev/pylint#9310.

Author: crazybolillo

astroid/interpreter/_import/spec.py

@@ -16,6 +16,7 @@
 import warnings
 import zipimport
 from collections.abc import Iterator, Sequence
+from functools import lru_cache
 from pathlib import Path
 from typing import Any, Literal, NamedTuple, Protocol
 
@@ -440,10 +441,15 @@ def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSp
     :return: A module spec, which describes how the module was
              found and where.
     """
+    return _find_spec(tuple(modpath), tuple(path) if path else None)
+
+
+@lru_cache(maxsize=1024)
+def _find_spec(modpath: tuple, path: tuple) -> ModuleSpec:
     _path = path or sys.path
 
     # Need a copy for not mutating the argument.
-    modpath = modpath[:]
+    modpath = list(modpath)
 
     submodule_path = None
     module_parts = modpath[:]
@@ -468,3 +474,7 @@ def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSp
             spec = spec._replace(submodule_search_locations=submodule_path)
 
     return spec
+
+
+def clear_spec_cache() -> None:
+    _find_spec.cache_clear()

astroid/manager.py

@@ -442,10 +442,12 @@ def clear_cache(self) -> None:
         # pylint: disable=import-outside-toplevel
         from astroid.brain.helpers import register_all_brains
         from astroid.inference_tip import clear_inference_tip_cache
+        from astroid.interpreter._import.spec import clear_spec_cache
         from astroid.interpreter.objectmodel import ObjectModel
         from astroid.nodes._base_nodes import LookupMixIn
         from astroid.nodes.scoped_nodes import ClassDef
 
+        clear_spec_cache()
         clear_inference_tip_cache()
         _invalidate_cache()  # inference context cache
 

tests/test_manager.py

@@ -23,6 +23,7 @@
     AttributeInferenceError,
 )
 from astroid.interpreter._import import util
+from astroid.interpreter._import.spec import clear_spec_cache
 from astroid.modutils import EXT_LIB_DIRS, module_in_path
 from astroid.nodes import Const
 from astroid.nodes.scoped_nodes import ClassDef, Module
@@ -41,6 +42,7 @@ class AstroidManagerTest(
 ):
     def setUp(self) -> None:
         super().setUp()
+        clear_spec_cache()
         self.manager = test_utils.brainless_manager()
 
     def test_ast_from_file(self) -> None:

tests/test_modutils.py

@@ -22,6 +22,7 @@
 from astroid import modutils
 from astroid.const import PY310_PLUS
 from astroid.interpreter._import import spec
+from astroid.interpreter._import.spec import clear_spec_cache
 
 from . import resources
 
@@ -41,6 +42,7 @@ class ModuleFileTest(unittest.TestCase):
     package = "mypypa"
 
     def tearDown(self) -> None:
+        clear_spec_cache()
         for k in list(sys.path_importer_cache):
             if "MyPyPa" in k:
                 del sys.path_importer_cache[k]