diff --git a/pyproject.toml b/pyproject.toml index 9446c67..ca1456c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ maintainers = [ ] dependencies = [ "importlib-metadata; python_version < '3.10'", + "platformdirs>=4.3,<5.0", + "tomli; python_version < '3.11'", "typing-extensions; python_version < '3.11'", ] @@ -44,8 +46,10 @@ dev = [ "ruff>=0.10,<1.0", ] test = [ + "deepdiff>=8.0,<9.0", "jsondiff>=2.2,<2.3", "hypothesis>=6.0.0,<7", + "parameterized>=0.9.0,<0.10", "pytest>=8.0.0,<9.0.0", "pytest-cov>=5.0.0,<6.0.0", "pytest-dotenv>=0.5.0,<1.0.0", @@ -53,7 +57,7 @@ test = [ "pytest-mock>=3.14.0,<4.0.0", "pytest-runner>=6.0.0,<7.0.0", "pytest-ordering>=0.6,<1.0.0", - "parameterized>=0.9.0,<0.10", + "tomli_w>=1.2,<1.3", ] [project.scripts] diff --git a/tests/models/test_configuration.py b/tests/models/test_configuration.py new file mode 100644 index 0000000..8513148 --- /dev/null +++ b/tests/models/test_configuration.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from variantlib.constants import VALIDATION_FEATURE_REGEX +from variantlib.constants import VALIDATION_NAMESPACE_REGEX +from variantlib.constants import VALIDATION_VALUE_REGEX +from variantlib.models.configuration import VariantConfiguration +from variantlib.models.variant import VariantFeature +from variantlib.models.variant import VariantProperty + + +def test_default_configuration(): + config = VariantConfiguration.default() + assert config.namespaces_priority == [] + assert config.features_priority == [] + assert config.property_priority == [] + + +@pytest.mark.parametrize( + "config_params", + [ + { + "namespaces": ["OmniCorp"], + "features": [], + "properties": [], + }, + { + "namespaces": ["OmniCorp"], + "features": ["OmniCorp::custom_feat"], + "properties": ["OmniCorp::custom_feat::secret_value"], + }, + { + "namespaces": ["OmniCorp", "AcmeCorp"], + "features": ["OmniCorp::custom_feat", "AcmeCorp :: custom_feat"], + "properties": [ + "OmniCorp :: custom_featA :: secret_value", + "OmniCorp :: custom_featB:: secret_value", + "AcmeCorp::custom_feat::secret_value", + ], + }, + ], +) +def test_from_toml_config(config_params: dict[str, list[str]]): + _ = VariantConfiguration.from_toml_config( + namespaces_priority=config_params["namespaces"], + features_priority=config_params["features"], + property_priority=config_params["properties"], + ) + + +@given(st.lists(st.from_regex(VALIDATION_NAMESPACE_REGEX))) +def test_namespaces_priority_validation(namespaces: list[str]): + config = VariantConfiguration(namespaces_priority=namespaces) + assert config.namespaces_priority == namespaces + + +@given( + st.lists(st.from_regex(VALIDATION_NAMESPACE_REGEX)), + st.lists( + st.builds( + VariantFeature, + namespace=st.just("OmniCorp"), + feature=st.from_regex(VALIDATION_FEATURE_REGEX), + ) + ), +) +def test_features_priority_validation( + namespaces: list[str], features: list[VariantFeature] +): + config = VariantConfiguration( + namespaces_priority=namespaces, features_priority=features + ) + assert config.features_priority == features + + +@given( + st.lists(st.from_regex(VALIDATION_NAMESPACE_REGEX)), + st.lists( + st.builds( + VariantFeature, + namespace=st.from_regex(VALIDATION_NAMESPACE_REGEX), + feature=st.from_regex(VALIDATION_FEATURE_REGEX), + ) + ), + st.lists( + st.builds( + VariantProperty, + namespace=st.from_regex(VALIDATION_NAMESPACE_REGEX), + feature=st.from_regex(VALIDATION_FEATURE_REGEX), + value=st.from_regex(VALIDATION_VALUE_REGEX), + ) + ), +) +def test_property_priority_validation( + namespaces: list[str], + features: list[VariantFeature], + properties: list[VariantProperty], +): + config = VariantConfiguration( + namespaces_priority=namespaces, + features_priority=features, + property_priority=properties, + ) + assert config.property_priority == properties diff --git a/tests/resolver/__init__.py b/tests/resolver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/resolver/test_filtering.py b/tests/resolver/test_filtering.py new file mode 100644 index 0000000..f7a5756 --- /dev/null +++ b/tests/resolver/test_filtering.py @@ -0,0 +1,491 @@ +from __future__ import annotations + +import copy +import random +from collections import deque + +import pytest + +from variantlib.errors import ValidationError +from variantlib.models.variant import VariantDescription +from variantlib.models.variant import VariantFeature +from variantlib.models.variant import VariantProperty +from variantlib.resolver.filtering import filter_variants_by_features +from variantlib.resolver.filtering import filter_variants_by_namespaces +from variantlib.resolver.filtering import filter_variants_by_property +from variantlib.resolver.filtering import remove_duplicates + + +@pytest.fixture(scope="session") +def vprops() -> list[VariantProperty]: + return [ + VariantProperty(namespace="OmniCorp", feature="custom_feat", value="value1"), + VariantProperty( + namespace="TyrellCorporation", feature="client_id", value="value2" + ), + ] + + +@pytest.fixture(scope="session") +def vdescs(vprops: list[VariantProperty]) -> list[VariantDescription]: + """Fixture to create a list of VariantDescription objects.""" + assert len(vprops) == 2 + vprop1, vprop2 = vprops + + return [ + VariantDescription([vprop1]), + VariantDescription([vprop2]), + VariantDescription([vprop1, vprop2]), + ] + + +# =========================== `remove_duplicates` =========================== # + + +def test_remove_duplicates(vdescs: list[VariantDescription]): + assert len(vdescs) == 3 + + # using `copy.deepcopy` to ensure that all objects are actually unique + input_vdescs = [copy.deepcopy(random.choice(vdescs)) for _ in range(100)] + filtered_vdescs = list(remove_duplicates(input_vdescs)) + + assert len(filtered_vdescs) == 3 + + for vdesc in vdescs: + assert vdesc in filtered_vdescs + + +def test_remove_duplicates_empty(): + assert list(remove_duplicates([])) == [] + + +@pytest.mark.parametrize( + "vdescs", + ["not a list", ["not a VariantDescription"]], +) +def test_remove_duplicates_validation_error(vdescs: list[VariantDescription]): + with pytest.raises(ValidationError): + deque(remove_duplicates(vdescs=vdescs), maxlen=0) + + +# ===================== `filter_variants_by_namespaces` ===================== # + + +def test_filter_variants_by_namespaces(vdescs: list[VariantDescription]): + assert len(vdescs) == 3 + vdesc1, vdesc2, _ = vdescs + + # No namespace forbidden - should return everything + assert ( + list( + filter_variants_by_namespaces( + vdescs=vdescs, + forbidden_namespaces=[], + ) + ) + == vdescs + ) + + # Non existing namespace forbidden - should return everything + assert ( + list( + filter_variants_by_namespaces( + vdescs=vdescs, + forbidden_namespaces=["NonExistentNamespace"], + ) + ) + == vdescs + ) + + # Only `OmniCorp` forbidden - should return `vdesc2` + assert list( + filter_variants_by_namespaces( + vdescs=vdescs, + forbidden_namespaces=["OmniCorp"], + ) + ) == [vdesc2] + + # Only `TyrellCorporation` forbidden - should return `vdesc1` + assert list( + filter_variants_by_namespaces( + vdescs=vdescs, + forbidden_namespaces=["TyrellCorporation"], + ) + ) == [vdesc1] + + # Both `OmniCorp` and `TyrellCorporation` forbidden - should return empty + # Note: Order should not matter + assert ( + list( + filter_variants_by_namespaces( + vdescs=vdescs, + forbidden_namespaces=["OmniCorp", "TyrellCorporation"], + ) + ) + == [] + ) + + assert ( + list( + filter_variants_by_namespaces( + vdescs=vdescs, + forbidden_namespaces=["TyrellCorporation", "OmniCorp"], + ) + ) + == [] + ) + + +@pytest.mark.parametrize( + ("vdescs", "forbidden_namespaces"), + [ + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + "not a list", + ), + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + [VariantProperty("not", "a", "str")], + ), + ("not a list", ["OmniCorp"]), + (["not a `VariantDescription`"], ["OmniCorp"]), + ], +) +def test_filter_variants_by_namespaces_validation_error( + vdescs: list[VariantDescription], forbidden_namespaces: list[str] +): + with pytest.raises(ValidationError): + deque( + filter_variants_by_namespaces( + vdescs=vdescs, + forbidden_namespaces=forbidden_namespaces, + ), + maxlen=0, + ) + + +# ====================== `filter_variants_by_features` ====================== # + + +def test_filter_variants_by_features( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + assert len(vprops) == 2 + vprop1, vprop2 = vprops + + assert len(vdescs) == 3 + vdesc1, vdesc2, _ = vdescs + + vfeat1 = vprop1.feature_object + vfeat2 = vprop2.feature_object + + # No feature forbidden - should return everything + assert ( + list( + filter_variants_by_features( + vdescs=vdescs, + forbidden_features=[], + ) + ) + == vdescs + ) + + # Non existing feature forbidden - should return everything + assert ( + list( + filter_variants_by_features( + vdescs=vdescs, + forbidden_features=[ + VariantFeature(namespace="UmbrellaCorporation", feature="AI") + ], + ) + ) + == vdescs + ) + + # Only `vfeat1` forbidden - should return `vdesc2` + assert list( + filter_variants_by_features( + vdescs=vdescs, + forbidden_features=[vfeat1], + ) + ) == [vdesc2] + + # Only `vfeat2` forbidden - should return `vdesc1` + assert list( + filter_variants_by_features( + vdescs=vdescs, + forbidden_features=[vfeat2], + ) + ) == [vdesc1] + + # Both of vfeats forbidden - should return empty + # Note: Order should not matter + assert ( + list( + filter_variants_by_features( + vdescs=vdescs, + forbidden_features=[vfeat1, vfeat2], + ) + ) + == [] + ) + + assert ( + list( + filter_variants_by_features( + vdescs=vdescs, + forbidden_features=[vfeat2, vfeat1], + ) + ) + == [] + ) + + +@pytest.mark.parametrize( + ("vdescs", "forbidden_features"), + [ + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + "not a list", + ), + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + ["not a `VariantFeature`"], + ), + ("not a list", VariantFeature("a", "b")), + (["not a `VariantDescription`"], VariantFeature("a", "b")), + ], +) +def test_filter_variants_by_features_validation_error( + vdescs: list[VariantDescription], forbidden_features: list[VariantFeature] +): + with pytest.raises(ValidationError): + deque( + filter_variants_by_features( + vdescs=vdescs, forbidden_features=forbidden_features + ), + maxlen=0, + ) + + +# ====================== `filter_variants_by_property` ====================== # + + +def test_filter_variants_by_property( + vdescs: list[VariantDescription], + vprops: list[VariantProperty], +): + assert len(vprops) == 2 + vprop1, vprop2 = vprops + + assert len(vdescs) == 3 + vdesc1, vdesc2, _ = vdescs + + # No property allowed - should return empty list + assert ( + list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[], + forbidden_properties=[], + ) + ) + == [] + ) + + # Non existing property allowed - should return empty list + assert ( + list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[ + VariantProperty( + namespace="UmbrellaCorporation", feature="AI", value="ChatBot" + ) + ], + forbidden_properties=[], + ) + ) + == [] + ) + + # Non existing property forbidden - should return empty list + assert ( + list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[], + forbidden_properties=[ + VariantProperty( + namespace="UmbrellaCorporation", feature="AI", value="ChatBot" + ) + ], + ) + ) + == [] + ) + + # Only `vprop1` allowed - should return `vdesc1` if not forbidden explicitly + assert list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop1], + forbidden_properties=[], + ) + ) == [vdesc1] + + assert ( + list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop1], + forbidden_properties=[vprop1], + ) + ) + == [] + ) + + assert list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop1], + forbidden_properties=[vprop2], + ) + ) == [vdesc1] + + # Only `vprop2` allowed - should return `vdesc2` if not forbidden explicitly + assert list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop2], + forbidden_properties=[], + ) + ) == [vdesc2] + + assert list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop2], + forbidden_properties=[vprop1], + ) + ) == [vdesc2] + + assert ( + list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop2], + forbidden_properties=[vprop2], + ) + ) + == [] + ) + + # Both of vprops - should return all `vdescs` if neither vprop1 or vprop2 is + # forbidden explictly + # Note: Order should not matter + assert ( + list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop1, vprop2], + forbidden_properties=[], + ) + ) + == vdescs + ) + + assert list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop1, vprop2], + forbidden_properties=[vprop1], + ) + ) == [vdesc2] + + assert list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop1, vprop2], + forbidden_properties=[vprop2], + ) + ) == [vdesc1] + + assert ( + list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop2, vprop1], + forbidden_properties=[], + ) + ) + == vdescs + ) + + assert list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop2, vprop1], + forbidden_properties=[vprop1], + ) + ) == [vdesc2] + + assert list( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=[vprop2, vprop1], + forbidden_properties=[vprop2], + ) + ) == [vdesc1] + + +@pytest.mark.parametrize( + ("vdescs", "allowed_properties", "forbidden_properties"), + [ + ( + "not a list", + [VariantProperty("a", "b", "c")], + [VariantProperty("a", "b", "c")], + ), + ( + [VariantProperty("not", "a", "VariantDescription")], + [VariantProperty("a", "b", "c")], + [VariantProperty("a", "b", "c")], + ), + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + "not a list", + [VariantProperty("a", "b", "c")], + ), + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + ["not a `VariantFeature`"], + [VariantProperty("a", "b", "c")], + ), + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + [VariantProperty("a", "b", "c")], + "not a list", + ), + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + [VariantProperty("a", "b", "c")], + ["not a `VariantFeature`"], + ), + ], +) +def test_filter_variants_by_property_validation_error( + vdescs: list[VariantDescription], + allowed_properties: list[VariantProperty], + forbidden_properties: list[VariantProperty], +): + with pytest.raises(ValidationError): + deque( + filter_variants_by_property( + vdescs=vdescs, + allowed_properties=allowed_properties, + forbidden_properties=forbidden_properties, + ), + maxlen=0, + ) diff --git a/tests/resolver/test_lib.py b/tests/resolver/test_lib.py new file mode 100644 index 0000000..1af3527 --- /dev/null +++ b/tests/resolver/test_lib.py @@ -0,0 +1,567 @@ +from __future__ import annotations + +import contextlib +import random + +import pytest +from deepdiff import DeepDiff + +from variantlib.errors import ValidationError +from variantlib.models.variant import VariantDescription +from variantlib.models.variant import VariantFeature +from variantlib.models.variant import VariantProperty +from variantlib.resolver.filtering import filter_variants_by_features +from variantlib.resolver.filtering import filter_variants_by_namespaces +from variantlib.resolver.filtering import filter_variants_by_property +from variantlib.resolver.filtering import remove_duplicates +from variantlib.resolver.lib import filter_variants +from variantlib.resolver.lib import sort_and_filter_supported_variants + + +def deep_diff( + a: list[VariantDescription], b: list[VariantDescription], ignore_ordering=False +) -> DeepDiff: + """Helper function to compare two objects using DeepDiff.""" + assert isinstance(a, list) + assert isinstance(b, list) + assert all(isinstance(vdesc, VariantDescription) for vdesc in a) + assert all(isinstance(vdesc, VariantDescription) for vdesc in b) + + return DeepDiff( + [vdesc.hexdigest for vdesc in a], + [vdesc.hexdigest for vdesc in b], + ignore_order=ignore_ordering, + ) + + +def shuffle_vdescs_with_duplicates( + vdescs: list[VariantDescription], +) -> list[VariantDescription]: + inputs_vdescs = [vdesc for vdesc in vdescs for _ in range(5)] + assert len(inputs_vdescs) == len(vdescs) * 5 + random.shuffle(inputs_vdescs) + return inputs_vdescs + + +@pytest.fixture(scope="session") +def vprops() -> list[VariantProperty]: + """Fixture to create a list of VariantProperty objects.""" + # This list assume sorting by feature and properties coming from the plugins + # Does not assume filtering per namespace. Only features & properties. + + return [ + # -------------------------- Plugin `OmniCorp` -------------------------- # + # Feature 1: `OmniCorp :: featA` + VariantProperty(namespace="OmniCorp", feature="featA", value="value"), + # Feature 2: `OmniCorp :: featB` + VariantProperty(namespace="OmniCorp", feature="featB", value="value"), + # ------------------------- Plugin `TyrellCorp` ------------------------- # + # Feature 1: `TyrellCorp :: featA` + VariantProperty(namespace="TyrellCorp", feature="featA", value="value"), + # Feature 2: `TyrellCorp :: featB` + # Property 2.1: `TyrellCorp :: featB :: abcde` + VariantProperty(namespace="TyrellCorp", feature="featB", value="abcde"), + # Property 2.2: `TyrellCorp :: featB :: efghij` + VariantProperty(namespace="TyrellCorp", feature="featB", value="efghij"), + # Feature 3: `TyrellCorp :: featC` + VariantProperty(namespace="TyrellCorp", feature="featC", value="value"), + ] + + +@pytest.fixture(scope="session") +def vdescs(vprops: list[VariantProperty]) -> list[VariantDescription]: + """Fixture to create a list of VariantDescription objects.""" + + assert len(vprops) == 6 + vprop1, vprop2, vprop3, vprop4, vprop5, vprop6 = vprops + + # fmt: off + # Important: vprop4 and vprop5 are mutually exclusive + return [ + # variants with 5 properties + VariantDescription([vprop1, vprop2, vprop3, vprop4, vprop6]), + VariantDescription([vprop1, vprop2, vprop3, vprop5, vprop6]), + + # variants with 4 properties + VariantDescription([vprop1, vprop2, vprop3, vprop4]), # - vprop6 + VariantDescription([vprop1, vprop2, vprop3, vprop5]), # - vprop6 + + VariantDescription([vprop1, vprop2, vprop3, vprop6]), # - vprop4/5 + + VariantDescription([vprop1, vprop2, vprop4, vprop6]), # - vprop3 + VariantDescription([vprop1, vprop2, vprop5, vprop6]), # - vprop3 + + VariantDescription([vprop1, vprop3, vprop4, vprop6]), # - vprop2 + VariantDescription([vprop1, vprop3, vprop5, vprop6]), # - vprop2 + + VariantDescription([vprop2, vprop3, vprop5, vprop6]), # - vprop1 + VariantDescription([vprop2, vprop3, vprop5, vprop6]), # - vprop1 + + # variants with 3 properties + # --- vprop1 --- # + VariantDescription([vprop1, vprop2, vprop3]), + VariantDescription([vprop1, vprop2, vprop4]), + VariantDescription([vprop1, vprop2, vprop5]), + VariantDescription([vprop1, vprop2, vprop6]), + + VariantDescription([vprop1, vprop3, vprop4]), + VariantDescription([vprop1, vprop3, vprop5]), + VariantDescription([vprop1, vprop3, vprop6]), + + VariantDescription([vprop1, vprop4, vprop6]), + VariantDescription([vprop1, vprop5, vprop6]), + + # --- vprop2 --- # + VariantDescription([vprop2, vprop3, vprop4]), + VariantDescription([vprop2, vprop3, vprop5]), + VariantDescription([vprop2, vprop3, vprop6]), + + VariantDescription([vprop2, vprop4, vprop6]), + VariantDescription([vprop2, vprop5, vprop6]), + + # --- vprop3 --- # + VariantDescription([vprop3, vprop4, vprop6]), + VariantDescription([vprop3, vprop5, vprop6]), + + # variants with 2 properties + # --- vprop1 --- # + VariantDescription([vprop1, vprop2]), + VariantDescription([vprop1, vprop3]), + VariantDescription([vprop1, vprop4]), + VariantDescription([vprop1, vprop5]), + VariantDescription([vprop1, vprop6]), + + # --- vprop2 --- # + VariantDescription([vprop2, vprop3]), + VariantDescription([vprop2, vprop4]), + VariantDescription([vprop2, vprop5]), + VariantDescription([vprop2, vprop6]), + + # --- vprop3 --- # + VariantDescription([vprop3, vprop4]), + VariantDescription([vprop3, vprop5]), + VariantDescription([vprop3, vprop6]), + + # --- vprop4 --- # + VariantDescription([vprop4, vprop6]), + + # --- vprop5 --- # + VariantDescription([vprop5, vprop6]), + + # variants with 1 property + VariantDescription([vprop1]), + VariantDescription([vprop2]), + VariantDescription([vprop3]), + VariantDescription([vprop4]), + VariantDescription([vprop5]), + VariantDescription([vprop6]), + ] + # fmt: on + + +# =========================== `filter_variants` =========================== # + + +def test_filter_variants_only_one_prop_allowed( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + assert len(vprops) == 6 + _, _, _, vprop4, _, _ = vprops + + inputs_vdescs = shuffle_vdescs_with_duplicates(vdescs=vdescs) + + assert ( + list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=[], + ) + ) + == [] + ) + + assert list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=[vprop4], + ) + ) == [VariantDescription([vprop4])] + + assert ( + list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=[vprop4], + forbidden_namespaces=[vprop4.namespace], + ) + ) + == [] + ) + + assert ( + list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=[vprop4], + forbidden_features=[vprop4.feature_object], + ) + ) + == [] + ) + + assert ( + list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=[vprop4], + forbidden_properties=[vprop4], + ) + ) + == [] + ) + + +def test_filter_variants_forbidden_feature_allowed_prop( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + assert len(vprops) == 6 + _, vprop2, _, vprop4, _, _ = vprops + + inputs_vdescs = shuffle_vdescs_with_duplicates(vdescs=vdescs) + + assert list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=[vprop4], + forbidden_features=[vprop2.feature_object], + ) + ) == [VariantDescription([vprop4])] + + +def test_filter_variants_forbidden_namespace_allowed_prop( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + assert len(vprops) == 6 + vprop1, _, _, vprop4, _, _ = vprops + + inputs_vdescs = shuffle_vdescs_with_duplicates(vdescs=vdescs) + + assert list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=[vprop4], + forbidden_namespaces=["NotExisting"], + ) + ) == [VariantDescription([vprop4])] + + assert list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=[vprop4], + forbidden_namespaces=[vprop1.namespace], + ) + ) == [VariantDescription([vprop4])] + + +def test_filter_variants_only_remove_duplicates( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + assert len(vprops) == 6 + + inputs_vdescs = shuffle_vdescs_with_duplicates(vdescs=vdescs) + + ddiff = deep_diff( + list(filter_variants(vdescs=inputs_vdescs, allowed_properties=vprops)), + vdescs, + ignore_ordering=True, + ) + assert ddiff == {}, ddiff + + ddiff = deep_diff( + list(remove_duplicates(vdescs=inputs_vdescs)), vdescs, ignore_ordering=True + ) + assert ddiff == {}, ddiff + + +def test_filter_variants_remove_duplicates_and_namespaces( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + assert len(vprops) == 6 + _, _, vprop3, vprop4, vprop5, vprop6 = vprops + + inputs_vdescs = shuffle_vdescs_with_duplicates(vdescs=vdescs) + + expected_vdescs = [ + # --- vprop3 --- # + VariantDescription([vprop3, vprop4, vprop6]), + VariantDescription([vprop3, vprop5, vprop6]), + # variants with 2 properties + # --- vprop3 --- # + VariantDescription([vprop3, vprop4]), + VariantDescription([vprop3, vprop5]), + VariantDescription([vprop3, vprop6]), + # --- vprop4 --- # + VariantDescription([vprop4, vprop6]), + # --- vprop5 --- # + VariantDescription([vprop5, vprop6]), + # variants with 1 property + VariantDescription([vprop3]), + VariantDescription([vprop4]), + VariantDescription([vprop5]), + VariantDescription([vprop6]), + ] + + ddiff = deep_diff( + list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=vprops, + forbidden_namespaces=["OmniCorp"], + ) + ), + expected_vdescs, + ignore_ordering=True, + ) + assert ddiff == {}, ddiff + + ddiff = deep_diff( + list( + filter_variants_by_namespaces( + remove_duplicates(vdescs=inputs_vdescs), + forbidden_namespaces=["OmniCorp"], + ) + ), + expected_vdescs, + ignore_ordering=True, + ) + assert ddiff == {}, ddiff + + +def test_filter_variants_remove_duplicates_and_features( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + assert len(vprops) == 6 + vprop1, vprop2, vprop3, vprop4, vprop5, vprop6 = vprops + + inputs_vdescs = shuffle_vdescs_with_duplicates(vdescs=vdescs) + + expected_vdescs = [ + # variants with 2 properties + # --- vprop1 --- # + VariantDescription([vprop1, vprop4]), + VariantDescription([vprop1, vprop5]), + # variants with 1 property + VariantDescription([vprop1]), + VariantDescription([vprop4]), + VariantDescription([vprop5]), + ] + + forbidden_features = [ + vprop2.feature_object, + vprop3.feature_object, + vprop6.feature_object, + ] + + ddiff = deep_diff( + list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=vprops, + forbidden_features=forbidden_features, + ) + ), + expected_vdescs, + ignore_ordering=True, + ) + assert ddiff == {}, ddiff + + ddiff = deep_diff( + list( + filter_variants_by_features( + remove_duplicates(vdescs=inputs_vdescs), + forbidden_features=forbidden_features, + ) + ), + expected_vdescs, + ignore_ordering=True, + ) + assert ddiff == {}, ddiff + + +def test_filter_variants_remove_duplicates_and_properties( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + assert len(vprops) == 6 + vprop1, vprop2, _, vprop4, vprop5, _ = vprops + + inputs_vdescs = shuffle_vdescs_with_duplicates(vdescs=vdescs) + + expected_vdescs = [ + # variants with 2 properties + # --- vprop1 --- # + VariantDescription([vprop1, vprop4]), + VariantDescription([vprop1, vprop5]), + # variants with 1 property + VariantDescription([vprop1]), + VariantDescription([vprop4]), + VariantDescription([vprop5]), + ] + + allowed_properties = [vprop1, vprop2, vprop4, vprop5] + + ddiff = deep_diff( + list( + filter_variants( + vdescs=inputs_vdescs, + allowed_properties=allowed_properties, + forbidden_properties=[vprop2], + ) + ), + expected_vdescs, + ignore_ordering=True, + ) + assert ddiff == {}, ddiff + + ddiff = deep_diff( + list( + filter_variants_by_property( + remove_duplicates(vdescs=inputs_vdescs), + allowed_properties=allowed_properties, + forbidden_properties=[vprop2], + ) + ), + expected_vdescs, + ignore_ordering=True, + ) + assert ddiff == {}, ddiff + + +# =================== `sort_and_filter_supported_variants` ================== # + + +def test_sort_and_filter_supported_variants( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + assert len(vprops) == 6 + + # Let's remove vprop2 [not supported] + vprop1, _, vprop3, vprop4, vprop5, vprop6 = vprops + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~ SORTING PARAMETERS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # + + # Top Priority: Anything that contains `vprop3` + prio_vprops: list[VariantProperty] = [vprop3] + + # Second Priority: Vprop4 and Vprop5 are prioritized priority (same feature) + # With vprop4 > vprop5 given that vprop4 is first in the list + prio_vfeats: list[VariantFeature] = [vprop4.feature_object] + + # Third Priority: namespaces + # vprop3, vprop4, vprop5, vprop6 [TyrellCorp] > vprop1, vprop2 ["OmniCorp"] # noqa: E501 + prio_namespaces = ["NotExistingNamespace", "TyrellCorp", "OmniCorp"] + + # Default Ordering: properties are assumed pre-sorted in features/properties + # vprop1 > vprop2 > vprop3 > vprop4 > vprop5 > vprop6 + # Note: Namespace is already accounted for in 3) + + # Last Preferential Order: More features are preferred over less features + + # ----------------------------------------------------------------------------- # + + # fmt: off + expected_vdescs = [ + # ============ A - Everything with vprop3 ============ # + # ============ A.1 - Everything with vprop3 & vprop4 ============ # + VariantDescription([vprop1, vprop3, vprop4, vprop6]), + VariantDescription([vprop3, vprop4, vprop6]), # TyrellCorp > OmniCorp + VariantDescription([vprop1, vprop3, vprop4]), # TyrellCorp > OmniCorp + VariantDescription([vprop3, vprop4]), + # ============ A.2 - Everything with vprop3 & vprop5 ============ # + VariantDescription([vprop1, vprop3, vprop5, vprop6]), + VariantDescription([vprop3, vprop5, vprop6]), # TyrellCorp > OmniCorp + VariantDescription([vprop1, vprop3, vprop5]), # TyrellCorp > OmniCorp + VariantDescription([vprop3, vprop5]), + # =========== A.4 - Everything with vprop3 & without vprop5/vprop6 =========== # + VariantDescription([vprop1, vprop3, vprop6]), + VariantDescription([vprop3, vprop6]), # TyrellCorp > OmniCorp + VariantDescription([vprop1, vprop3]), # TyrellCorp > OmniCorp + # =========== A.5 - vprop3 alone =========== # + VariantDescription([vprop3]), + + # ============ B - Everything without vprop3 ============ # + # ============ B.1 - Everything without vprop3 & with vprop4 ============ # + VariantDescription([vprop1, vprop4, vprop6]), + VariantDescription([vprop4, vprop6]), # TyrellCorp > OmniCorp + VariantDescription([vprop1, vprop4]), # TyrellCorp > OmniCorp + VariantDescription([vprop4]), + + # ============ B.2 - Everything without vprop3 & with vprop5 ============ # + VariantDescription([vprop1, vprop5, vprop6]), + VariantDescription([vprop5, vprop6]), # TyrellCorp > OmniCorp + VariantDescription([vprop1, vprop5]), # TyrellCorp > OmniCorp + VariantDescription([vprop5]), + + # == C - Everything without vprop3/vprop4/vprop5 and TyrellCorp > OmniCorp == # + VariantDescription([vprop1, vprop6]), + VariantDescription([vprop6]), + VariantDescription([vprop1]), + ] + # fmt: on + + inputs_vdescs = shuffle_vdescs_with_duplicates(vdescs=vdescs) + + ddiff = deep_diff( + sort_and_filter_supported_variants( + vdescs=inputs_vdescs, + supported_vprops=[vprop1, vprop3, vprop4, vprop5, vprop6], + property_priorities=prio_vprops, + feature_priorities=prio_vfeats, + namespace_priorities=prio_namespaces, + ), + expected_vdescs, + ) + assert ddiff == {}, ddiff + + +# # =================== `Validation Testing` ================== # + + +@pytest.mark.parametrize( + ("vdescs", "vprops"), + [ + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + "not a list", + ), + ( + [VariantDescription([VariantProperty("a", "b", "c")])], + [VariantFeature("not_a", "VariantProperty")], + ), + ("not a list", VariantProperty("a", "b", "c")), + (["not a `VariantDescription`"], VariantProperty("a", "b", "c")), + ], +) +def test_sort_and_filter_supported_variants_validation_errors( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + feature_priorities = [] + with contextlib.suppress(TypeError, AttributeError): + feature_priorities = list({vprop.feature_object for vprop in vprops}) + + with pytest.raises(ValidationError): + sort_and_filter_supported_variants( + vdescs=vdescs, + supported_vprops=vprops, + feature_priorities=feature_priorities, + ) + + +def test_sort_and_filter_supported_variants_validation_errors_with_no_priority( + vdescs: list[VariantDescription], vprops: list[VariantProperty] +): + # This one specifies no ordering/priority => can't sort + with pytest.raises(ValidationError, match="has no priority"): + sort_and_filter_supported_variants( + vdescs=vdescs, + supported_vprops=vprops, + ) diff --git a/tests/resolver/test_sorting.py b/tests/resolver/test_sorting.py new file mode 100644 index 0000000..bb75f2e --- /dev/null +++ b/tests/resolver/test_sorting.py @@ -0,0 +1,413 @@ +from __future__ import annotations + +import sys + +import pytest + +from variantlib.errors import ValidationError +from variantlib.models.variant import VariantDescription +from variantlib.models.variant import VariantFeature +from variantlib.models.variant import VariantProperty +from variantlib.resolver.sorting import get_feature_priority +from variantlib.resolver.sorting import get_namespace_priority +from variantlib.resolver.sorting import get_property_priority +from variantlib.resolver.sorting import get_variant_property_priority_tuple +from variantlib.resolver.sorting import sort_variant_properties +from variantlib.resolver.sorting import sort_variants_descriptions + +# ========================= get_property_priority =========================== # + + +def test_get_property_priority(): + vprop1 = VariantProperty(namespace="OmniCorp", feature="feat", value="value1") + vprop2 = VariantProperty(namespace="OmniCorp", feature="feat", value="value2") + vprop3 = VariantProperty(namespace="OmniCorp", feature="feat", value="value3") + assert get_property_priority(vprop1, [vprop1, vprop2]) == 0 + assert get_property_priority(vprop2, [vprop1, vprop2]) == 1 + assert get_property_priority(vprop1, [vprop1, vprop2, vprop3]) == 0 + assert get_property_priority(vprop2, [vprop1, vprop2, vprop3]) == 1 + + +def test_negative_get_property_priority(): + vprop1 = VariantProperty(namespace="OmniCorp", feature="feat", value="value1") + vprop2 = VariantProperty(namespace="OmniCorp", feature="feat", value="value2") + + assert get_property_priority(vprop1, None) == sys.maxsize + assert get_property_priority(vprop1, []) == sys.maxsize + assert get_property_priority(vprop1, [vprop2]) == sys.maxsize + + +@pytest.mark.parametrize( + ("vprop", "property_priorities"), + [ + ("not a `VariantProperty`", None), + (VariantProperty("a", "b", "c"), "not a list or None"), + (VariantProperty("a", "b", "c"), VariantProperty("not", "a", "list")), + (VariantProperty("a", "b", "c"), [{"not a VariantProperty": True}]), + ], +) +def test_get_property_priority_validation_error( + vprop: VariantProperty, property_priorities: list[VariantProperty] | None +): + with pytest.raises(ValidationError): + get_property_priority(vprop=vprop, property_priorities=property_priorities) + + +# ========================== get_feature_priority =========================== # + + +def test_get_feature_priority(): + vprop1 = VariantProperty(namespace="OmniCorp", feature="feature", value="value") + vprop2 = VariantProperty(namespace="OmniCorp", feature="other_feat", value="value") + feature_priorities = [ + VariantFeature(namespace="OmniCorp", feature="feature"), + VariantFeature(namespace="OmniCorp", feature="other_feat"), + ] + assert get_feature_priority(vprop1, feature_priorities) == 0 + assert get_feature_priority(vprop2, feature_priorities) == 1 + + +def test_negative_get_feature_priority(): + vprop = VariantProperty(namespace="OmniCorp", feature="no_exist", value="value") + vfeat = VariantFeature(namespace="OmniCorp", feature="feature") + + assert get_feature_priority(vprop, None) == sys.maxsize + assert get_feature_priority(vprop, []) == sys.maxsize + assert get_feature_priority(vprop, [vfeat]) == sys.maxsize + + +@pytest.mark.parametrize( + ("vprop", "feature_priorities"), + [ + ("not a `VariantProperty`", None), + (VariantProperty("a", "b", "c"), "not a list or None"), + (VariantProperty("a", "b", "c"), VariantFeature("not_a", "list")), + (VariantProperty("a", "b", "c"), [{"not a VariantFeature": True}]), + ], +) +def test_get_feature_priority_validation_error( + vprop: VariantProperty, feature_priorities: list[VariantFeature] | None +): + with pytest.raises(ValidationError): + get_feature_priority(vprop=vprop, feature_priorities=feature_priorities) + + +# ========================= get_namespace_priority ========================== # + + +def test_get_namespace_priority(): + vprop1 = VariantProperty(namespace="OmniCorp", feature="feature", value="value") + vprop2 = VariantProperty(namespace="OtherCorp", feature="feature", value="value") + vprop3 = VariantProperty(namespace="NoCorp", feature="feature", value="value") + namespace_priorities = ["OmniCorp", "OtherCorp"] + assert get_namespace_priority(vprop1, namespace_priorities) == 0 + assert get_namespace_priority(vprop2, namespace_priorities) == 1 + assert get_namespace_priority(vprop3, namespace_priorities) == sys.maxsize + + +def test_negative_get_namespace_priority(): + vprop = VariantProperty(namespace="OmniCorp", feature="no_exist", value="value") + + assert get_namespace_priority(vprop, None) == sys.maxsize + assert get_namespace_priority(vprop, []) == sys.maxsize + assert get_namespace_priority(vprop, ["OtherCorp"]) == sys.maxsize + + +@pytest.mark.parametrize( + ("vprop", "namespace_priorities"), + [ + ("not a `VariantProperty`", None), + (VariantProperty("a", "b", "c"), "not a list or None"), + (VariantProperty("a", "b", "c"), [{"not a str": True}]), + ], +) +def test_get_namespace_priority_validation_error( + vprop: VariantProperty, namespace_priorities: list[str] | None +): + with pytest.raises(ValidationError): + get_namespace_priority(vprop=vprop, namespace_priorities=namespace_priorities) + + +# =================== get_variant_property_priority_tuple =================== # + + +def test_get_variant_property_priority_tuple(): + vprop = VariantProperty(namespace="OmniCorp", feature="custom_feat", value="value1") + property_priorities = [ + VariantProperty(namespace="OtherCorp", feature="other_feat", value="value2"), + vprop, + ] + feature_priorities = [ + VariantFeature(namespace=vprop.namespace, feature=vprop.feature), + VariantFeature(namespace="OmniCorp", feature="feature"), + ] + namespace_priorities = ["OtherCorp"] + assert get_variant_property_priority_tuple( + vprop, namespace_priorities, feature_priorities, property_priorities + ) == (1, 0, sys.maxsize) + + +@pytest.mark.parametrize( + ("vprop", "namespace_priorities", "feature_priorities", "property_priorities"), + [ + ("not a `VariantProperty`", None, None, None), + (VariantProperty("a", "b", "c"), "not a list or None", None, None), + ( + VariantProperty("a", "b", "c"), + [VariantProperty("not", "a", "str")], + None, + None, + ), + (VariantProperty("a", "b", "c"), None, "not a list or None", None), + (VariantProperty("a", "b", "c"), None, ["not a VariantFeature"], None), + (VariantProperty("a", "b", "c"), None, None, "not a list or None"), + (VariantProperty("a", "b", "c"), None, None, ["not a VariantProperty"]), + ], +) +def test_get_variant_property_priority_tuple_validation_error( + vprop: VariantProperty, + namespace_priorities: list[str] | None, + feature_priorities: list[VariantFeature] | None, + property_priorities: list[VariantProperty] | None, +): + with pytest.raises(ValidationError): + get_variant_property_priority_tuple( + vprop=vprop, + namespace_priorities=namespace_priorities, + feature_priorities=feature_priorities, + property_priorities=property_priorities, + ) + + +# ========================= sort_variant_properties ========================= # + + +def test_sort_variant_properties(): + vprop_list = [ + VariantProperty(namespace="OmniCorp", feature="featA", value="value"), + VariantProperty(namespace="OmniCorp", feature="featB", value="value1"), + VariantProperty(namespace="OmniCorp", feature="featB", value="value2"), + VariantProperty(namespace="OmniCorp", feature="featC", value="value"), + VariantProperty(namespace="OmniCorp", feature="featD", value="value"), + VariantProperty(namespace="OtherCorp", feature="featA", value="value"), + VariantProperty(namespace="OtherCorp", feature="featB", value="value1"), + VariantProperty(namespace="OtherCorp", feature="featB", value="value2"), + VariantProperty(namespace="OtherCorp", feature="featC", value="value"), + VariantProperty(namespace="OtherCorp", feature="featD", value="value"), + ] + property_priorities = [ + VariantProperty(namespace="OtherCorp", feature="featA", value="value"), + VariantProperty(namespace="OmniCorp", feature="featA", value="value"), + VariantProperty(namespace="OmniCorp", feature="featC", value="value"), + VariantProperty(namespace="OtherCorp", feature="featC", value="value"), + ] + feature_priorities = [ + VariantFeature(namespace="OtherCorp", feature="featB"), + VariantFeature(namespace="OmniCorp", feature="featB"), + ] + namespace_priorities = ["OmniCorp", "OtherCorp"] + sorted_vprops = sort_variant_properties( + vprop_list, namespace_priorities, feature_priorities, property_priorities + ) + assert sorted_vprops == [ + # sorted by property priorities + VariantProperty(namespace="OtherCorp", feature="featA", value="value"), + VariantProperty(namespace="OmniCorp", feature="featA", value="value"), + VariantProperty(namespace="OmniCorp", feature="featC", value="value"), + VariantProperty(namespace="OtherCorp", feature="featC", value="value"), + # sorted by feature priorities + VariantProperty(namespace="OtherCorp", feature="featB", value="value1"), + VariantProperty(namespace="OtherCorp", feature="featB", value="value2"), + VariantProperty(namespace="OmniCorp", feature="featB", value="value1"), + VariantProperty(namespace="OmniCorp", feature="featB", value="value2"), + # sorted by namespace priorities + VariantProperty(namespace="OmniCorp", feature="featD", value="value"), + VariantProperty(namespace="OtherCorp", feature="featD", value="value"), + ] + + +@pytest.mark.parametrize( + ("vprops", "namespace_priorities", "feature_priorities", "property_priorities"), + [ + ("not a list of `VariantProperty`", None, None, None), + (VariantProperty("not", "a", "list"), None, None, None), + (["not a `VariantProperty`"], None, None, None), + ([VariantProperty("a", "b", "c")], "not a list or None", None, None), + ( + [VariantProperty("a", "b", "c")], + [VariantProperty("not", "a", "str")], + None, + None, + ), + ([VariantProperty("a", "b", "c")], None, "not a list or None", None), + ([VariantProperty("a", "b", "c")], None, ["not a VariantFeature"], None), + ([VariantProperty("a", "b", "c")], None, None, "not a list or None"), + ([VariantProperty("a", "b", "c")], None, None, ["not a VariantProperty"]), + # Can't sort without any ordering priorities + ([VariantProperty("a", "b", "c")], None, None, None), + ], +) +def test_sort_variant_properties_validation_error( + vprops: list[VariantProperty], + namespace_priorities: list[str] | None, + feature_priorities: list[VariantFeature] | None, + property_priorities: list[VariantProperty] | None, +): + with pytest.raises(ValidationError): + sort_variant_properties( + vprops=vprops, + namespace_priorities=namespace_priorities, + feature_priorities=feature_priorities, + property_priorities=property_priorities, + ) + + +# ========================= sort_variants_descriptions ========================= # + + +def test_sort_variants_descriptions(): + vprops_proprioty_list = [ + VariantProperty(namespace="OmniCorp", feature="featA", value="value"), + VariantProperty(namespace="OmniCorp", feature="featB", value="value1"), + VariantProperty(namespace="OmniCorp", feature="featB", value="value2"), + VariantProperty(namespace="OmniCorp", feature="featC", value="value"), + VariantProperty(namespace="OmniCorp", feature="featD", value="value"), + VariantProperty(namespace="OtherCorp", feature="featA", value="value"), + VariantProperty(namespace="OtherCorp", feature="featB", value="value1"), + VariantProperty(namespace="OtherCorp", feature="featB", value="value2"), + VariantProperty(namespace="OtherCorp", feature="featC", value="value"), + VariantProperty(namespace="OtherCorp", feature="featD", value="value"), + VariantProperty(namespace="AnyCorp", feature="feature", value="value"), + ] + + vdesc1 = VariantDescription( + [ + VariantProperty(namespace="OtherCorp", feature="featA", value="value"), + VariantProperty(namespace="OmniCorp", feature="featB", value="value1"), + VariantProperty(namespace="OmniCorp", feature="featC", value="value"), + VariantProperty(namespace="OtherCorp", feature="featC", value="value"), + ] + ) + vdesc2 = VariantDescription( + [VariantProperty(namespace="OtherCorp", feature="featA", value="value")] + ) + vdesc3 = VariantDescription( + [VariantProperty(namespace="OmniCorp", feature="featA", value="value")] + ) + + assert sort_variants_descriptions( + vdescs=[vdesc1], property_priorities=vprops_proprioty_list + ) == [vdesc1] + + assert sort_variants_descriptions( + vdescs=[vdesc1, vdesc2], property_priorities=vprops_proprioty_list + ) == [vdesc1, vdesc2] + + # order permutation + assert sort_variants_descriptions( + vdescs=[vdesc2, vdesc1], property_priorities=vprops_proprioty_list + ) == [vdesc1, vdesc2] + + assert sort_variants_descriptions( + vdescs=[vdesc1, vdesc3], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc1] + + # order permutation + assert sort_variants_descriptions( + vdescs=[vdesc3, vdesc1], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc1] + + assert sort_variants_descriptions( + vdescs=[vdesc2, vdesc3], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc2] + + # order permutation + assert sort_variants_descriptions( + vdescs=[vdesc3, vdesc2], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc2] + + assert sort_variants_descriptions( + vdescs=[vdesc1, vdesc2, vdesc3], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc1, vdesc2] + + # order permutation + assert sort_variants_descriptions( + vdescs=[vdesc2, vdesc1, vdesc3], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc1, vdesc2] + + # order permutation + assert sort_variants_descriptions( + vdescs=[vdesc1, vdesc3, vdesc2], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc1, vdesc2] + + # order permutation + assert sort_variants_descriptions( + vdescs=[vdesc2, vdesc3, vdesc1], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc1, vdesc2] + + # order permutation + assert sort_variants_descriptions( + vdescs=[vdesc3, vdesc2, vdesc1], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc1, vdesc2] + + # order permutation + assert sort_variants_descriptions( + vdescs=[vdesc3, vdesc1, vdesc2], property_priorities=vprops_proprioty_list + ) == [vdesc3, vdesc1, vdesc2] + + +@pytest.mark.parametrize( + "vdesc", + [ + VariantDescription( + properties=[ + VariantProperty( + namespace="OmniCorp", feature="feat", value="other_value" + ) + ] + ), + VariantDescription( + properties=[ + VariantProperty(namespace="OmniCorp", feature="feat", value="value"), + VariantProperty( + namespace="OmniCorp", feature="other_feat", value="other_value" + ), + ], + ), + ], +) +def test_sort_variants_descriptions_ranking_validation_error(vdesc: VariantDescription): + vprops = [VariantProperty(namespace="OmniCorp", feature="feat", value="value")] + + # Test with a completely different property (same feature, different value) + with pytest.raises(ValidationError, match="Filtering should be applied first."): + sort_variants_descriptions( + vdescs=[vdesc], + property_priorities=vprops, + ) + + +@pytest.mark.parametrize( + ("vdescs", "property_priorities"), + [ + ("not a list", [VariantProperty("a", "b", "c")]), + (["not a VariantDescription"], [VariantProperty("a", "b", "c")]), + (VariantDescription([VariantProperty("a", "b", "c")]), "not a list or None"), + ( + VariantDescription([VariantProperty("a", "b", "c")]), + VariantProperty("not", "a", "list"), + ), + ( + VariantDescription([VariantProperty("a", "b", "c")]), + [{"not a VariantProperty": True}], + ), + ], +) +def test_sort_variants_descriptions_validation_error( + vdescs: list[VariantDescription], property_priorities: list[VariantProperty] +): + with pytest.raises(ValidationError): + sort_variants_descriptions( + vdescs=vdescs, + property_priorities=property_priorities, + ) diff --git a/tests/test_configuration.py b/tests/test_configuration.py new file mode 100644 index 0000000..c005130 --- /dev/null +++ b/tests/test_configuration.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +import platformdirs +import tomli_w + +from variantlib.configuration import ConfigEnvironments +from variantlib.configuration import VariantConfiguration +from variantlib.configuration import get_configuration_files +from variantlib.constants import CONFIG_FILENAME +from variantlib.models.configuration import VariantConfiguration as ConfigurationModel +from variantlib.models.variant import VariantFeature +from variantlib.models.variant import VariantProperty + + +def test_reset(): + VariantConfiguration._config = ConfigurationModel.default() # noqa: SLF001 + assert VariantConfiguration._config is not None # noqa: SLF001 + VariantConfiguration.reset() + assert VariantConfiguration._config is None # noqa: SLF001 + + +def test_get_configuration_files(): + config_files = get_configuration_files() + assert config_files[ConfigEnvironments.LOCAL] == Path.cwd() / CONFIG_FILENAME + assert ( + config_files[ConfigEnvironments.VIRTUALENV] + == Path(sys.prefix) / CONFIG_FILENAME + ) + assert ( + config_files[ConfigEnvironments.USER] + == Path( + platformdirs.user_config_dir("variantlib", appauthor=False, roaming=True) + ) + / CONFIG_FILENAME + ) + assert ( + config_files[ConfigEnvironments.GLOBAL] + == Path(platformdirs.site_config_dir("variantlib", appauthor=False)) + / CONFIG_FILENAME + ) + + +@patch("variantlib.configuration.get_configuration_files") +def test_get_default_config_with_no_file(mock_get_config_files): + mock_get_config_files.return_value = { + ConfigEnvironments.LOCAL: Path("/nonexistent/config.toml"), + ConfigEnvironments.VIRTUALENV: Path("/nonexistent/config.toml"), + ConfigEnvironments.USER: Path("/nonexistent/config.toml"), + ConfigEnvironments.GLOBAL: Path("/nonexistent/config.toml"), + } + config = VariantConfiguration.get_config() + assert config == ConfigurationModel.default() + + +@patch("variantlib.configuration.get_configuration_files") +def test_get_config_from_file(mock_get_config_files): + data = { + "property_priority": [ + "fictional_hw::architecture::mother", + "fictional_tech::risk_exposure::25", + ], + "features_priority": [ + "fictional_hw::architecture", + "fictional_tech::risk_exposure", + "simd_x86_64::feature3", + ], + "namespaces_priority": [ + "fictional_hw", + "fictional_tech", + "simd_x86_64", + "non_existent_provider", + ], + } + with tempfile.NamedTemporaryFile(mode="w+", suffix=".toml") as temp_file: + temp_file.write(tomli_w.dumps(data)) + temp_file.flush() + + def _get_config_files() -> dict[ConfigEnvironments, Path]: + return { + ConfigEnvironments.LOCAL: Path("/nonexistent/config.toml"), + ConfigEnvironments.VIRTUALENV: Path("/nonexistent/config.toml"), + ConfigEnvironments.USER: Path("/nonexistent/config.toml"), + ConfigEnvironments.GLOBAL: Path("/nonexistent/config.toml"), + } + + features_priority = [ + VariantFeature.from_str(f) for f in data["features_priority"] + ] + + property_priority = [ + VariantProperty.from_str(f) for f in data["property_priority"] + ] + + for env in ConfigEnvironments: + config_files = _get_config_files() + config_files[env] = Path(temp_file.name) + mock_get_config_files.return_value = config_files + + config = VariantConfiguration.get_config() + assert config.features_priority == features_priority + assert config.property_priority == property_priority + assert config.namespaces_priority == data["namespaces_priority"] + + +def test_class_properties_with_default(): + VariantConfiguration.reset() + assert VariantConfiguration._config is None # noqa: SLF001 + assert VariantConfiguration.namespaces_priority == [] + assert VariantConfiguration._config is not None # noqa: SLF001 + + VariantConfiguration.reset() + assert VariantConfiguration._config is None # noqa: SLF001 + assert VariantConfiguration.features_priority == [] + assert VariantConfiguration._config is not None # noqa: SLF001 + + VariantConfiguration.reset() + assert VariantConfiguration._config is None # noqa: SLF001 + assert VariantConfiguration.property_priority == [] + assert VariantConfiguration._config is not None # noqa: SLF001 diff --git a/variantlib/commands/generate_index_json.py b/variantlib/commands/generate_index_json.py index c625711..2e911dd 100644 --- a/variantlib/commands/generate_index_json.py +++ b/variantlib/commands/generate_index_json.py @@ -17,7 +17,7 @@ logger.setLevel(logging.INFO) -def generate_index_json(args: list[str]) -> None: # noqa: PLR0912 +def generate_index_json(args: list[str]) -> None: parser = argparse.ArgumentParser( prog="generate_index_json", description="Generate a JSON index of all package variants", diff --git a/variantlib/configuration.py b/variantlib/configuration.py new file mode 100644 index 0000000..3fc036f --- /dev/null +++ b/variantlib/configuration.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import logging +import sys +from enum import IntEnum +from functools import cache +from functools import wraps +from pathlib import Path +from typing import TYPE_CHECKING +from typing import Any +from typing import Callable +from typing import TypeVar + +import platformdirs + +from variantlib import errors +from variantlib.constants import CONFIG_FILENAME +from variantlib.models.configuration import VariantConfiguration as ConfigurationModel +from variantlib.utils import classproperty + +if TYPE_CHECKING: + from variantlib.models.variant import VariantFeature + from variantlib.models.variant import VariantProperty + +if sys.version_info >= (3, 11): + from typing import Self + + import tomllib +else: + import tomli as tomllib + from typing_extensions import Self + +logger = logging.getLogger(__name__) + + +class ConfigEnvironments(IntEnum): + LOCAL = 1 + VIRTUALENV = 2 + USER = 3 + GLOBAL = 4 + + +@cache +def get_configuration_files() -> dict[ConfigEnvironments, Path]: + return { + ConfigEnvironments.LOCAL: Path.cwd() / CONFIG_FILENAME, + ConfigEnvironments.VIRTUALENV: Path(sys.prefix) / CONFIG_FILENAME, + ConfigEnvironments.USER: ( + Path( + platformdirs.user_config_dir( + "variantlib", appauthor=False, roaming=True + ) + ) + / CONFIG_FILENAME + ), + ConfigEnvironments.GLOBAL: ( + Path(platformdirs.site_config_dir("variantlib", appauthor=False)) + / CONFIG_FILENAME + ), + } + + +R = TypeVar("R") + + +def check_initialized(func: Callable[..., R]) -> Callable[..., R]: + @wraps(func) + def wrapper(cls: type[VariantConfiguration], *args: Any, **kwargs: Any) -> R: + if cls._config is None: + cls._config = cls.get_config() + + return func(cls, *args, **kwargs) + + return wrapper + + +class VariantConfiguration: + _config: ConfigurationModel | None = None + + def __new__(cls, *args: Any, **kwargs: dict[str, Any]) -> Self: + raise RuntimeError(f"Cannot instantiate {cls.__name__}") + + @classmethod + def reset(cls) -> None: + """Reset the configuration to its initial state""" + cls._config = None + + @classmethod + def get_config(cls) -> ConfigurationModel: + """Load the configuration from the configuration files""" + # TODO: Read namespace priority configuration + # TODO: Read namespace-feature prority configuration + # TODO: Read namespace-feature-value prority configuration + config_files = get_configuration_files() + + for config_name in ConfigEnvironments: + if (cfg_f := config_files[config_name]).exists(): + logger.info("Loading configuration file: %s", config_files[config_name]) + with cfg_f.open("rb") as f: + try: + config = tomllib.load(f) + except tomllib.TOMLDecodeError as e: + raise errors.ConfigurationError from e + + return ConfigurationModel.from_toml_config(**config) + + # No user-configuration file found + return ConfigurationModel.default() + + @classproperty + @check_initialized + def namespaces_priority(cls) -> list[str]: # noqa: N805 + return cls._config.namespaces_priority # type: ignore[union-attr] + + @classproperty + @check_initialized + def features_priority(cls) -> list[VariantFeature]: # noqa: N805 + return cls._config.features_priority # type: ignore[union-attr] + + @classproperty + @check_initialized + def property_priority(cls) -> list[VariantProperty]: # noqa: N805 + return cls._config.property_priority # type: ignore[union-attr] diff --git a/variantlib/constants.py b/variantlib/constants.py index af591f3..31319a7 100644 --- a/variantlib/constants.py +++ b/variantlib/constants.py @@ -3,7 +3,7 @@ import re VARIANT_HASH_LEN = 8 -CONFIG_FILENAME = "wheelvariant.toml" +CONFIG_FILENAME = "variants.toml" VALIDATION_NAMESPACE_REGEX = r"^[A-Za-z0-9_]+$" VALIDATION_FEATURE_REGEX = r"^[A-Za-z0-9_]+$" diff --git a/variantlib/errors.py b/variantlib/errors.py index 6a63f18..5ce60f4 100644 --- a/variantlib/errors.py +++ b/variantlib/errors.py @@ -12,3 +12,7 @@ class PluginMissingError(RuntimeError): class InvalidVariantEnvSpecError(ValueError): """Environment specifier for variants is invalid""" + + +class ConfigurationError(ValueError): + pass diff --git a/variantlib/models/configuration.py b/variantlib/models/configuration.py new file mode 100644 index 0000000..9b7d26a --- /dev/null +++ b/variantlib/models/configuration.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import sys +from dataclasses import dataclass +from dataclasses import field + +from variantlib.constants import VALIDATION_NAMESPACE_REGEX +from variantlib.models.base import BaseModel +from variantlib.models.validators import validate_and +from variantlib.models.validators import validate_list_matches_re +from variantlib.models.validators import validate_type +from variantlib.models.variant import VariantFeature +from variantlib.models.variant import VariantProperty + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +@dataclass(frozen=True) +class VariantConfiguration(BaseModel): + """ + Configuration class for variantlib. + + This class is used to define the configuration for the variantlib library. + It includes fields for the namespace, feature, and value, along with validation + checks for each field. + + # Sorting Note: First is best. + + Attributes: + namespaces_priority (list): Sorted list of "variant namespaces" by priority. + features_priority (list): Sorted list of `VariantFeature` by priority. + property_priority (list): Sorted list of `VariantProperty` by priority. + """ + + namespaces_priority: list[str] = field( + metadata={ + "validator": lambda val: validate_and( + [ + lambda v: validate_type(v, list[str]), + lambda v: validate_list_matches_re(v, VALIDATION_NAMESPACE_REGEX), + ], + value=val, + ) + } + ) + + features_priority: list[VariantFeature] = field( + metadata={ + "validator": lambda val: validate_and( + [ + lambda v: validate_type(v, list[VariantFeature]), + ], + value=val, + ) + }, + default_factory=list, + ) + + property_priority: list[VariantProperty] = field( + metadata={ + "validator": lambda val: validate_and( + [ + lambda v: validate_type(v, list[VariantProperty]), + ], + value=val, + ) + }, + default_factory=list, + ) + + @classmethod + def default(cls) -> Self: + """ + Create a default `VariantConfiguration` instance. + + Returns: + VariantConfiguration: A new instance with default values. + """ + + # TODO: Verify the default values make sense + + return cls( + namespaces_priority=[], + features_priority=[], + property_priority=[], + ) + + @classmethod + def from_toml_config( + cls, + namespaces_priority: list[str] | None = None, + features_priority: list[str] | None = None, + property_priority: list[str] | None = None, + ) -> Self: + """ + Create a Configuration instance from TOML-based configuration. + + Returns: + Configuration: A new Configuration instance. + """ + + # Convert the `features_priority: list[str]` into `list[VariantFeature]` + _features_priority: list[VariantFeature] = [] + if features_priority is not None: + for vfeat in features_priority: + validate_type(vfeat, str) + _features_priority.append(VariantFeature.from_str(vfeat)) + + # Convert the `property_priority: list[str]` into `list[VariantProperty]` + _property_priority: list[VariantProperty] = [] + if property_priority is not None: + for vprop in property_priority: + validate_type(vprop, str) + _property_priority.append(VariantProperty.from_str(vprop)) + + return cls( + namespaces_priority=namespaces_priority or [], + features_priority=_features_priority, + property_priority=_property_priority, + ) diff --git a/variantlib/models/validators.py b/variantlib/models/validators.py index e9fbacb..163528c 100644 --- a/variantlib/models/validators.py +++ b/variantlib/models/validators.py @@ -6,6 +6,7 @@ from types import GenericAlias from typing import Any from typing import Callable +from typing import Protocol from typing import Union from typing import get_args from typing import get_origin @@ -107,8 +108,10 @@ def validate_and(validators: list[Callable], value: Any) -> None: def _validate_type(value: Any, expected_type: type) -> type | None: if isinstance(expected_type, GenericAlias): list_type = get_origin(expected_type) + if not isinstance(value, list_type): return type(value) + if list_type is dict: assert isinstance(value, dict) key_type, value_type = get_args(expected_type) @@ -122,6 +125,7 @@ def _validate_type(value: Any, expected_type: type) -> type | None: key_ored = Union.__getitem__((key_type, *incorrect_key_types)) value_ored = Union.__getitem__((value_type, *incorrect_value_types)) return list_type[key_ored, value_ored] # type: ignore[index] + else: (item_type,) = get_args(expected_type) assert isinstance(value, Iterable) @@ -130,8 +134,16 @@ def _validate_type(value: Any, expected_type: type) -> type | None: if incorrect_types: ored = Union.__getitem__((item_type, *incorrect_types)) return list_type[ored] # type: ignore[index] - elif not isinstance(value, expected_type): + + # Protocols and Iterable must enable subclassing to pass + elif issubclass(expected_type, (Protocol, Iterable)): # type: ignore[arg-type] + if not isinstance(value, expected_type): + return type(value) + + # Do not use isinstance here - we want to reject subclasses + elif type(value) is not expected_type: return type(value) + return None diff --git a/variantlib/models/variant.py b/variantlib/models/variant.py index e56389f..1bd9866 100644 --- a/variantlib/models/variant.py +++ b/variantlib/models/variant.py @@ -7,6 +7,7 @@ from dataclasses import asdict from dataclasses import dataclass from dataclasses import field +from functools import cached_property from variantlib.constants import VALIDATION_FEATURE_REGEX from variantlib.constants import VALIDATION_NAMESPACE_REGEX @@ -51,7 +52,7 @@ class VariantFeature(BaseModel): } ) - @property + @cached_property def feature_hash(self) -> int: # __class__ is being added to guarantee the hash to be specific to this class # note: can't use `self.__class__` because of inheritance @@ -84,7 +85,7 @@ def from_str(cls, input_str: str) -> Self: if match is None: raise ValidationError( - f"Invalid format: {input_str}. Expected format: " + f"Invalid format: `{input_str}`, expected format: " "' :: '" ) @@ -110,11 +111,15 @@ class VariantProperty(VariantFeature): } ) - @property + @cached_property def property_hash(self) -> int: # __class__ is being added to guarantee the hash to be specific to this class return hash((self.__class__, self.namespace, self.feature, self.value)) + @cached_property + def feature_object(self) -> VariantFeature: + return VariantFeature(namespace=self.namespace, feature=self.feature) + def to_str(self) -> str: # Variant: :: :: return f"{self.namespace} :: {self.feature} :: {self.value}" @@ -133,8 +138,8 @@ def from_str(cls, input_str: str) -> Self: if match is None: raise ValidationError( - f"Invalid format: {input_str}. " - "Expected format: ' :: :: '" + f"Invalid format: `{input_str}`, " + "expected format: ` :: :: `" ) # Extract the namespace, feature, and value from the match groups @@ -184,7 +189,7 @@ def __post_init__(self) -> None: # Execute the validator super().__post_init__() - @property + @cached_property def hexdigest(self) -> str: """ Compute the hash of the object. @@ -225,12 +230,12 @@ def is_valid(self, allow_unknown_plugins: bool = True) -> bool: allow_unknown_plugins or None not in self.results.values() ) - @property + @cached_property def invalid_properties(self) -> list[VariantProperty]: """List of properties declared invalid by plugins""" return [x for x, y in self.results.items() if y is False] - @property + @cached_property def unknown_properties(self) -> list[VariantProperty]: """List of properties not in any recognized namespace""" return [x for x, y in self.results.items() if y is None] diff --git a/variantlib/resolver/__init__.py b/variantlib/resolver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/variantlib/resolver/filtering.py b/variantlib/resolver/filtering.py new file mode 100644 index 0000000..6e97cc1 --- /dev/null +++ b/variantlib/resolver/filtering.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +import logging +from collections.abc import Iterable +from typing import TYPE_CHECKING + +from variantlib.models.validators import validate_type +from variantlib.models.variant import VariantDescription +from variantlib.models.variant import VariantFeature +from variantlib.models.variant import VariantProperty + +if TYPE_CHECKING: + from collections.abc import Generator + +logger = logging.getLogger(__name__) + + +def remove_duplicates( + vdescs: Iterable[VariantDescription], +) -> Generator[VariantDescription]: + # Input validation + validate_type(vdescs, Iterable) + + seen = set() + + def _should_include(vdesc: VariantDescription) -> bool: + """ + Check if any of the namespaces in the variant description are not allowed. + """ + validate_type(vdesc, VariantDescription) + + if vdesc.hexdigest in seen: + logger.info( + "Variant `%(vhash)s` has been removed because it is a duplicate", + {"vhash": vdesc.hexdigest}, + ) + return False + + seen.add(vdesc.hexdigest) + return True + + yield from filter(_should_include, vdescs) + + +def filter_variants_by_namespaces( + vdescs: Iterable[VariantDescription], + forbidden_namespaces: list[str], +) -> Generator[VariantDescription]: + """ + Filters out `VariantDescription` that contain any unsupported variant namespace. + + ** Implementation Note:** + - Installer will provide a list of forbidden namespaces by the user. + + :param vdescs: list of `VariantDescription` to filter. + :param forbidden_namespaces: List of forbidden variant namespaces as `str`. + :return: Filtered list of `VariantDescription`. + """ + + if forbidden_namespaces is None: + forbidden_namespaces = [] + + # Input validation + validate_type(vdescs, Iterable) + validate_type(forbidden_namespaces, list[str]) + + # Note: for performance reasons we convert the list to a set to avoid O(n) lookups + _forbidden_namespaces = set(forbidden_namespaces) + + def _should_include(vdesc: VariantDescription) -> bool: + """ + Check if any of the namespaces in the variant description are not allowed. + """ + validate_type(vdesc, VariantDescription) + + if forbidden_vprops := [ + vprop + for vprop in vdesc.properties + if vprop.namespace in _forbidden_namespaces + ]: + logger.info( + "Variant `%(vhash)s` has been rejected because one or many of the " + "variant namespaces `[%(vprops)s]` have been explicitly rejected.", + { + "vhash": vdesc.hexdigest, + "vprops": ", ".join([vprop.to_str() for vprop in forbidden_vprops]), + }, + ) + return False + + return True + + yield from filter(_should_include, vdescs) + + +def filter_variants_by_features( + vdescs: Iterable[VariantDescription], + forbidden_features: list[VariantFeature], +) -> Generator[VariantDescription]: + """ + Filters out `VariantDescription` that contain any unsupported variant feature. + + ** Implementation Note:** + - Installer will provide a list of forbidden VariantFeature by the user. + + :param vdescs: list of `VariantDescription` to filter. + :param forbidden_features: List of forbidden `VariantFeature`. + :return: Filtered list of `VariantDescription`. + """ + + if forbidden_features is None: + forbidden_features = [] + + # Input validation + validate_type(vdescs, Iterable) + validate_type(forbidden_features, list[VariantFeature]) + + # for performance reasons we convert the list to a set to avoid O(n) lookups + forbidden_feature_hexs = {vfeat.feature_hash for vfeat in forbidden_features} + + def _should_include(vdesc: VariantDescription) -> bool: + """ + Check if any of the VariantFeatures in the variant description are not allowed. + """ + validate_type(vdesc, VariantDescription) + + if forbidden_vprops := [ + vprop + for vprop in vdesc.properties + if vprop.feature_hash in forbidden_feature_hexs + ]: + logger.info( + "Variant `%(vhash)s` has been rejected because one or many of the " + "variant features `[%(vprops)s]` have been explicitly rejected.", + { + "vhash": vdesc.hexdigest, + "vprops": ", ".join([vprop.to_str() for vprop in forbidden_vprops]), + }, + ) + return False + + return True + + yield from filter(_should_include, vdescs) + + +def filter_variants_by_property( + vdescs: Iterable[VariantDescription], + allowed_properties: list[VariantProperty], + forbidden_properties: list[VariantProperty] | None = None, +) -> Generator[VariantDescription]: + """ + Filters out `VariantDescription` that contain any unsupported variant property. + + ** Implementation Note:** + - Installer will provide the list of allowed properties from variant provider + plugins. + - User can [optionally] provide a list of forbidden properties to be excluded. + Forbidden properties take precedence of "allowed properties" and will be excluded. + + :param vdescs: list of `VariantDescription` to filter. + :param allowed_properties: List of allowed `VariantProperty`. + :param forbidden_properties: List of forbidden `VariantProperty`. + :return: Filtered list of `VariantDescription`. + """ + + if forbidden_properties is None: + forbidden_properties = [] + + # Input validation + validate_type(vdescs, Iterable) + validate_type(allowed_properties, list[VariantProperty]) + validate_type(forbidden_properties, list[VariantProperty]) + + # for performance reasons we convert the list to a set to avoid O(n) lookups + allowed_properties_hexs = {vfeat.property_hash for vfeat in allowed_properties} + forbidden_properties_hexs = {vfeat.property_hash for vfeat in forbidden_properties} + + def _should_include(vdesc: VariantDescription) -> bool: + """ + Check if any of the namespaces in the variant description are not allowed. + """ + validate_type(vdesc, VariantDescription) + + if forbidden_vprops := [ + vprop + for vprop in vdesc.properties + if ( + vprop.property_hash not in allowed_properties_hexs + or vprop.property_hash in forbidden_properties_hexs + ) + ]: + logger.info( + "Variant `%(vhash)s` has been rejected because one or many of the " + "variant properties `[%(vprops)s]` are not supported or have been " + "explicitly rejected.", + { + "vhash": vdesc.hexdigest, + "vprops": ", ".join([vprop.to_str() for vprop in forbidden_vprops]), + }, + ) + return False + + return True + + yield from filter(_should_include, vdescs) diff --git a/variantlib/resolver/lib.py b/variantlib/resolver/lib.py new file mode 100644 index 0000000..a9b041d --- /dev/null +++ b/variantlib/resolver/lib.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from variantlib.models.validators import validate_type +from variantlib.models.variant import VariantDescription +from variantlib.models.variant import VariantFeature +from variantlib.models.variant import VariantProperty +from variantlib.resolver.filtering import filter_variants_by_features +from variantlib.resolver.filtering import filter_variants_by_namespaces +from variantlib.resolver.filtering import filter_variants_by_property +from variantlib.resolver.filtering import remove_duplicates +from variantlib.resolver.sorting import sort_variant_properties +from variantlib.resolver.sorting import sort_variants_descriptions + +if TYPE_CHECKING: + from collections.abc import Generator + + +def filter_variants( + vdescs: list[VariantDescription], + allowed_properties: list[VariantProperty], + forbidden_namespaces: list[str] | None = None, + forbidden_features: list[VariantFeature] | None = None, + forbidden_properties: list[VariantProperty] | None = None, +) -> Generator[VariantDescription]: + """ + Filters out a `list` of `VariantDescription` with the following filters: + - Duplicates removed + - Only allowed `variant properties` kept + + # Optionally: + - Forbidden `variant namespaces` removed - if `forbidden_namespaces` is not None + - Forbidden `variant features` removed - if `forbidden_features` is not None + - Forbidden `variant properties` removed - if `forbidden_properties` is not None + + :param vdescs: list of `VariantDescription` to filter. + :param allowed_properties: List of allowed `VariantProperty`. + :param forbidden_namespaces: List of forbidden variant namespaces as `str`. + :param forbidden_features: List of forbidden `VariantFeature`. + :param forbidden_properties: List of forbidden `VariantProperty`. + :return: Filtered list of `VariantDescription`. + """ + + # Input validation + validate_type(vdescs, list[VariantDescription]) + validate_type(allowed_properties, list[VariantProperty]) + + if forbidden_namespaces is not None: + validate_type(forbidden_namespaces, list[str]) + + if forbidden_features is not None: + validate_type(forbidden_features, list[VariantFeature]) + + if forbidden_properties is not None: + validate_type(forbidden_properties, list[VariantProperty]) + + # Step 1 + # Remove duplicates - There should never be any duplicates on the index + # - filename collision (same filename & same hash) + # - hash collision inside `variants.json` + # => Added for safety and to avoid any potential bugs + # (Note: In all fairness, even if it was to happen, it would most + # likely not be a problem given that we just pick the best match) + result = remove_duplicates(vdescs) + + # Step 2 [Optional] + # Remove any `VariantDescription` which declares any `VariantProperty` with + # a variant namespace explicitly forbidden by the user. + if forbidden_namespaces is not None: + result = filter_variants_by_namespaces( + vdescs=result, + forbidden_namespaces=forbidden_namespaces, + ) + + # Step 3 [Optional] + # Remove any `VariantDescription` which declares any `VariantProperty` with + # `namespace :: feature` (aka. `VariantFeature`) explicitly forbidden by the user. + if forbidden_features is not None: + result = filter_variants_by_features( + vdescs=result, + forbidden_features=forbidden_features, + ) + + # Step 4 [Optional] + # Remove any `VariantDescription` which declare any `VariantProperty` + # `namespace :: feature :: value` unsupported on this platform or explicitly + # forbidden by the user. + if allowed_properties is not None: + result = filter_variants_by_property( + vdescs=result, + allowed_properties=allowed_properties, + forbidden_properties=forbidden_properties, + ) + + yield from result + + +def sort_and_filter_supported_variants( + vdescs: list[VariantDescription], + supported_vprops: list[VariantProperty], + forbidden_namespaces: list[str] | None = None, + forbidden_features: list[VariantFeature] | None = None, + forbidden_properties: list[VariantProperty] | None = None, + namespace_priorities: list[str] | None = None, + feature_priorities: list[VariantFeature] | None = None, + property_priorities: list[VariantProperty] | None = None, +) -> list[VariantDescription]: + """ + Sort and filter a list of `VariantDescription` objects based on their + `VariantProperty`s. + + :param vdescs: List of `VariantDescription` objects. + :param supported_vprops: List of `VariantProperty` objects supported on the platform + :param namespace_priorities: Ordered list of `str` objects. + :param feature_priorities: Ordered list of `VariantFeature` objects. + :param property_priorities: Ordered list of `VariantProperty` objects. + :return: Sorted and filtered list of `VariantDescription` objects. + """ + validate_type(vdescs, list[VariantDescription]) + + if supported_vprops is None: + """No supported properties provided, return no variants.""" + return [] + + validate_type(supported_vprops, list[VariantProperty]) + + # Step 1: we remove any duplicate, or unsupported `VariantDescription` on + # this platform. + filtered_vdescs = list( + filter_variants( + vdescs=vdescs, + allowed_properties=supported_vprops, + forbidden_namespaces=forbidden_namespaces, + forbidden_features=forbidden_features, + forbidden_properties=forbidden_properties, + ) + ) + + # Step 2: we sort the supported `VariantProperty`s based on their respective + # priority. + sorted_supported_vprops = sort_variant_properties( + vprops=supported_vprops, + property_priorities=property_priorities, + feature_priorities=feature_priorities, + namespace_priorities=namespace_priorities, + ) + + # Step 3: we sort the `VariantDescription` based on the sorted supported properties + # and their respective priority. + return sort_variants_descriptions( + filtered_vdescs, + property_priorities=sorted_supported_vprops, + ) diff --git a/variantlib/resolver/sorting.py b/variantlib/resolver/sorting.py new file mode 100644 index 0000000..a4606c3 --- /dev/null +++ b/variantlib/resolver/sorting.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import logging +import sys + +from variantlib.errors import ValidationError +from variantlib.models.validators import validate_type +from variantlib.models.variant import VariantDescription +from variantlib.models.variant import VariantFeature +from variantlib.models.variant import VariantProperty + +logger = logging.getLogger(__name__) + + +def get_property_priority( + vprop: VariantProperty, + property_priorities: list[VariantProperty] | None, +) -> int: + """ + Get the property priority of a `VariantProperty` object. + + :param vprop: `VariantProperty` object. + :param property_priorities: ordered list of `VariantProperty` objects. + :return: Property priority of the `VariantProperty` object. + """ + validate_type(vprop, VariantProperty) + + if property_priorities is None: + return sys.maxsize + validate_type(property_priorities, list[VariantProperty]) + + _property_priorities = [vprop.property_hash for vprop in property_priorities] + + # if not present push at the end + try: + return _property_priorities.index(vprop.property_hash) + except ValueError: + return sys.maxsize + + +def get_feature_priority( + vprop: VariantProperty, + feature_priorities: list[VariantFeature] | None, +) -> int: + """ + Get the feature priority of a `VariantProperty` object. + + :param vprop: `VariantProperty` object. + :param feature_priorities: ordered list of `VariantFeature` objects. + :return: Feature priority of the `VariantProperty` object. + """ + validate_type(vprop, VariantProperty) + + if feature_priorities is None: + return sys.maxsize + validate_type(feature_priorities, list[VariantFeature]) + + _feature_priorities = [vfeat.feature_hash for vfeat in feature_priorities] + + # if not present push at the end + try: + return _feature_priorities.index(vprop.feature_hash) + except ValueError: + return sys.maxsize + + +def get_namespace_priority( + vprop: VariantProperty, + namespace_priorities: list[str] | None, +) -> int: + """ + Get the namespace priority of a `VariantProperty` object. + + :param vprop: `VariantProperty` object. + :param namespace_priorities: ordered list of `str` objects. + :return: Namespace priority of the `VariantProperty` object. + """ + validate_type(vprop, VariantProperty) + + if namespace_priorities is None: + return sys.maxsize + validate_type(namespace_priorities, list[str]) + + # if not present push at the end + try: + return namespace_priorities.index(vprop.namespace) + except ValueError: + return sys.maxsize + + +def get_variant_property_priority_tuple( + vprop: VariantProperty, + namespace_priorities: list[str] | None, + feature_priorities: list[VariantFeature] | None, + property_priorities: list[VariantProperty] | None, +) -> tuple[int, int, int]: + """ + Get the variant property priority of a `VariantProperty` object. + + :param vprop: `VariantProperty` object. + :param namespace_priorities: ordered list of `str` objects. + :param feature_priorities: ordered list of `VariantFeature` objects. + :param property_priorities: ordered list of `VariantProperty` objects. + :return: Variant property priority of the `VariantProperty` object. + """ + validate_type(vprop, VariantProperty) + + ranking_tuple = ( + # First Priority + get_property_priority(vprop, property_priorities), + # Second Priority + get_feature_priority(vprop, feature_priorities), + # Third Priority + get_namespace_priority(vprop, namespace_priorities), + ) + + if all(x == sys.maxsize for x in ranking_tuple): + raise ValidationError( + f"VariantProperty {vprop} has no priority - this should not happen." + ) + + return ranking_tuple + + +def sort_variant_properties( + vprops: list[VariantProperty], + namespace_priorities: list[str] | None, + feature_priorities: list[VariantFeature] | None, + property_priorities: list[VariantProperty] | None, +) -> list[VariantProperty]: + """ + Sort a list of `VariantProperty` objects based on their priority. + + :param vprops: List of `VariantProperty` objects. + :param namespace_priorities: ordered list of `str` objects. + :param feature_priorities: ordered list of `VariantFeature` objects. + :param property_priorities: ordered list of `VariantProperty` objects. + :return: Sorted list of `VariantProperty` objects. + """ + validate_type(vprops, list[VariantProperty]) + + return sorted( + vprops, + key=lambda x: get_variant_property_priority_tuple( + x, namespace_priorities, feature_priorities, property_priorities + ), + ) + + +def sort_variants_descriptions( + vdescs: list[VariantDescription], property_priorities: list[VariantProperty] +) -> list[VariantDescription]: + """ + Sort a list of `VariantDescription` objects based on their `VariantProperty`s. + + :param vdescs: List of `VariantDescription` objects. + :param property_priorities: ordered list of `VariantProperty` objects. + :return: Sorted list of `VariantDescription` objects. + """ + validate_type(vdescs, list[VariantDescription]) + validate_type(property_priorities, list[VariantProperty]) + + # Pre-compute the property hash for the property priorities + # This is used to speed up the sorting process. + # The property_hash is used to compare the `VariantProperty` objects + property_hash_priorities = [vprop.property_hash for vprop in property_priorities] + + def _get_rank_tuple(vdesc: VariantDescription) -> tuple[int, ...]: + """ + Get the rank tuple of a `VariantDescription` object. + + :param vdesc: `VariantDescription` object. + :return: Rank tuple[int, ...] of the `VariantDescription` object. + """ + + # --------------------------- Implementation Notes --------------------------- # + # - `property_hash_priorities` is ordered. It's a list. + # - `vdesc_prop_hashes` is unordered. It's a set. + # + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Performance Notes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # + # * Only `property_hash_priorities` needs to be ordered. The set is used for + # performance reasons. + # * `vdesc.properties` hashes are pre-computed and saved to avoid recomputing + # them multiple times. + # * `property_priorities` are also pre-hashed to avoid recomputing them + # ---------------------------------------------------------------------------- # + + # using a set for performance reason: O(1) access time. + vdesc_prop_hashes = {vprop.property_hash for vprop in vdesc.properties} + + # N-dimensional tuple with tuple[N] of 1 or sys.maxsize + # 1 if the property is present in the `VariantDescription` object, + # sys.maxsize if not present. + # This is used to sort the `VariantDescription` objects based on their + # `VariantProperty`s. + ranking_tuple = tuple( + 1 if vprop_hash in vdesc_prop_hashes else sys.maxsize + for vprop_hash in property_hash_priorities + ) + + if sum(1 for x in ranking_tuple if x != sys.maxsize) != len(vdesc.properties): + raise ValidationError( + f"VariantDescription {vdesc} contains properties not in the property " + "priorities list - this should not happen. Filtering should be applied " + "first." + ) + + return ranking_tuple + + return sorted(vdescs, key=_get_rank_tuple) diff --git a/variants.dist.toml b/variants.dist.toml new file mode 100644 index 0000000..95ba5e5 --- /dev/null +++ b/variants.dist.toml @@ -0,0 +1,52 @@ +# ======================= EXAMPLE `variants.toml` FILE ====================== # + +# ============= Top-Most Priority ============= # + +# 1. Define the priority of variant properties - 1st order priority. +# => Expected format: "namespace::feature::value" +# +# OPTIONAL: In most cases - users will not specify the following. Only used when the user wants +# to modify the default VariantProperty priority ordering. +# +# Note: This is a lazy-list: Only specify the ones you want to "bump up" to the top of the list + +property_priority = [ + "fictional_hw::architecture::mother", + "fictional_tech::risk_exposure:25", +] + +# ============= Second-Most Priority ============= # + +# 2. Define the priority of variant features - 2nd order priority under the variant properties. +# => Expected format: "namespace::feature" +# +# OPTIONAL: In most cases - users will not specify the following. Only used when the user wants +# to modify the default VariantFeature priority ordering. +# +# Note: This is a lazy-list: Only specify the ones you want to "bump up" to the top of the list + +features_priority = [ + "fictional_hw::architecture", + "fictional_tech::risk_exposure", + "simd_x86_64::feature3", +] + +# ============= Default Priority Ordering ============= # + +# 3. Define the priority of variant namespaces +# => Expected format: "namespace" +# +# MANDATORY AND IMPORTANT - PLEASE READ ! +# - As long as there is more than 1 variant plugin installed. This field must be specified. +# => no default ordering is assumed. +# +# - If len(plugins) > 1 and an installed plugin is missing in the priority list => raise ConfigurationError +# +# - If namespaces is specified by no plugin uses it => warning issued. + +namespaces_priority = [ + "fictional_hw", + "fictional_tech", + "simd_x86_64", + "non_existent_provider", +]