diff --git a/conan/internal/graph/graph_binaries.py b/conan/internal/graph/graph_binaries.py index 2ddc28ad172..5890cfb9149 100644 --- a/conan/internal/graph/graph_binaries.py +++ b/conan/internal/graph/graph_binaries.py @@ -168,13 +168,15 @@ def _find_existing_compatible_binaries(self, node, compatibles, remotes, update) self._compatible_found(conanfile, package_id, compatible_package) return if not should_update_reference(conanfile.ref, update): - conanfile.output.info(f"Compatible configurations not found in cache, checking servers") + remotes_to_check = self._filter_compat_remotes(node, remotes) + conanfile.output.info("Compatible configurations not found in cache, " + f"checking {len(remotes_to_check)} servers") for package_id, compatible_package in compatibles.items(): conanfile.output.info(f"'{package_id}': " f"{conanfile.info.dump_diff(compatible_package)}") node._package_id = package_id # Modifying package id under the hood, FIXME node.binary = None # Invalidate it - self._evaluate_download(node, remotes, update) + self._evaluate_download(node, remotes_to_check, update) if node.binary == BINARY_DOWNLOAD: self._compatible_found(conanfile, package_id, compatible_package) return @@ -199,6 +201,24 @@ def _find_build_compatible_binary(self, node, compatibles): node.binary = original_binary node._package_id = original_package_id + def _filter_compat_remotes(self, node, remotes): + # For each node, check if the remote contains the recipe, + # so that we can skip checking for packages in remotes that don't have the recipe + # TODO: Cache this, but make sure to reset when changing remotes + remotes_to_check = [] + for remote in remotes: + try: + self._remote_manager.get_recipe_revision_reference(node.ref, remote) + remotes_to_check.append(remote) + except NotFoundException: + continue + except ConanConnectionError: + node.conanfile.output.error(f"Failed checking for recipe '{node.ref}' in remote " + f"'{remote.name}': remote not available") + raise + + return remotes_to_check + def _evaluate_node(self, node, build_mode, remotes, update): assert node.binary is None, "Node.binary should be None" assert node.package_id is not None, "Node.package_id shouldn't be None" diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 0e88687f0af..331591cdd2e 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -285,7 +285,26 @@ def get_latest_package_reference(self, pref, remote, info=None) -> PkgReference: def get_recipe_revision_reference(self, ref, remote) -> bool: assert ref.revision is not None, "recipe_exists needs a revision" - return self._call_remote(remote, "get_recipe_revision_reference", ref) + + cached_method = remote._caching.setdefault("get_recipe_revision_reference", {}) + try: + result = cached_method[ref] + except KeyError: + try: + result = self._call_remote(remote, "get_recipe_revision_reference", ref) + cached_method[ref] = result + return result + except NotFoundException as e: + # Let's avoid leaking memory by saving all the exception objects, + # which translates to a ~2x memory increase. Now, it only saves the type and the + # final message. For now, let's cache only the NotFoundException one. + cached_method[ref] = self._ErrorMsg(str(e)) + raise e + else: + if isinstance(result, self._ErrorMsg): + # Let's raise it + raise NotFoundException(result.message) + return result def get_package_revision_reference(self, pref, remote) -> bool: assert pref.revision is not None, "get_package_revision_reference needs a revision" diff --git a/test/integration/package_id/compatible_test.py b/test/integration/package_id/compatible_test.py index 9de6c48d52a..4dd9085bdde 100644 --- a/test/integration/package_id/compatible_test.py +++ b/test/integration/package_id/compatible_test.py @@ -43,8 +43,8 @@ def package_info(self): assert "pkg/0.1@user/stable: PackageInfo!: Gcc version: 4.9!" in client.out client.assert_listed_binary({"pkg/0.1@user/stable": ("1ded27c9546219fbd04d4440e05b2298f8230047", "Build")}) - assert "lib/0.1@user/stable: Compatible configurations not found in cache, checking servers" not in client.out - assert "pkg/0.1@user/stable: Compatible configurations not found in cache, checking servers" in client.out + assert "lib/0.1@user/stable: Compatible configurations not found in cache, checking 0 servers" not in client.out + assert "pkg/0.1@user/stable: Compatible configurations not found in cache, checking 0 servers" in client.out def test_compatible_setting_no_user_channel(self): client = TestClient() @@ -724,3 +724,30 @@ def libc_compat(conanfile): tc.run("install --requires=dep/1.0 -s=libc_version=3 -s=compiler.cppstd=14") assert f"dep/1.0: Found compatible package '{dep_package_id}': compiler.cppstd=17, " \ f"libc_version=2" in tc.out + + +def test_check_server_non_existing_recipes(): + c = TestClient(default_server_user=True) + compatibles = textwrap.dedent("""\ + def compatibility(conanfile): + return [{"settings": [("arch", v)]} for v in ("x86", "x86_64", "armv7", "armv8")] + """) + c.save_home({"extensions/plugins/compatibility/compatibility.py": compatibles}) + + conanfile = GenConanfile("dep", "0.1").with_settings("arch") + c.save({"dep/conanfile.py": conanfile, + "consumer/conanfile.py": GenConanfile().with_requires("dep/0.1")}) + + c.run(f"export dep") + + c.run(f"install --requires=dep/0.1 -s arch=x86", assert_error=True) + + assert "dep/0.1: Checking 3 compatible configurations" in c.out + assert "dep/0.1: Compatible configurations not found in cache, checking 0 servers" in c.out + + c.run(f"upload dep/0.1 -r=default -c") + + c.run(f"install --requires=dep/0.1 -s arch=x86", assert_error=True) + + assert "dep/0.1: Checking 3 compatible configurations" in c.out + assert "dep/0.1: Compatible configurations not found in cache, checking 1 servers" in c.out diff --git a/test/integration/package_id/test_cache_compatibles.py b/test/integration/package_id/test_cache_compatibles.py index 2642915f4df..216143c7e03 100644 --- a/test/integration/package_id/test_cache_compatibles.py +++ b/test/integration/package_id/test_cache_compatibles.py @@ -149,7 +149,7 @@ def compatibility(conanfile): c.run(f"install consumer {base_settings} -s compiler.cppstd=17") assert "dep/0.1: Checking 3 compatible configurations" in c.out - assert "dep/0.1: Compatible configurations not found in cache, checking servers" in c.out + assert "dep/0.1: Compatible configurations not found in cache, checking 1 servers" in c.out assert "dep/0.1: Main binary package '6179018ccb6b15e6443829bf3640e25f2718b931' missing" \ in c.out assert "Found compatible package '326c500588d969f55133fdda29506ef61ef03eee': " \ @@ -159,7 +159,7 @@ def compatibility(conanfile): # second time, not download, already in cache c.run(f"install consumer {base_settings} -s compiler.cppstd=17") assert "dep/0.1: Checking 3 compatible configurations" in c.out - assert "dep/0.1: Compatible configurations not found in cache, checking servers" not in c.out + assert "dep/0.1: Compatible configurations not found in cache, checking 1 servers" not in c.out assert "dep/0.1: Main binary package '6179018ccb6b15e6443829bf3640e25f2718b931' missing" in c.out assert "Found compatible package '326c500588d969f55133fdda29506ef61ef03eee': " \ "compiler.cppstd=20" in c.out @@ -173,7 +173,7 @@ def compatibility(conanfile): c.run(f"install consumer {base_settings} -s compiler.cppstd=17 --update") assert "dep/0.1: Checking 3 compatible configurations" in c.out - assert "dep/0.1: Compatible configurations not found in cache, checking servers" not in c.out + assert "dep/0.1: Compatible configurations not found in cache, checking 1 servers" not in c.out assert "dep/0.1: Main binary package '6179018ccb6b15e6443829bf3640e25f2718b931' missing" in c.out assert "Found compatible package 'ce92fac7c26ace631e30875ddbb3a58a190eb601': " \ "compiler.cppstd=14" in c.out