Support for foolbox attacks via a simple interface

README.md

@@ -138,6 +138,10 @@ This package makes use of the following tools and libraries:
 
 - **auto-verify** ([GitHub](https://github.com/ADA-research/auto-verify))
     - For integrating verifiers [nnenum](https://github.com/stanleybak/nnenum), [AB-Crown](https://github.com/Verified-Intelligence/alpha-beta-CROWN), [VeriNet](https://github.com/vas-group-imperial/VeriNet), and [Oval-Bab](https://github.com/oval-group/oval-bab). Please refer to the auto-verify [documentation](https://ada-research.github.io/auto-verify/) for details about auto-verify.
+
+- **foolbox** ([GitHub](https://github.com/bethgelab/foolbox))
+  - Rauber, J., Brendel, W., and Bethge, M., "Foolbox: A Python toolbox to benchmark the robustness of machine learning models," in *Reliable Machine Learning in the Wild Workshop, 34th International Conference on Machine Learning*, 2017. [Online]. Available: http://arxiv.org/abs/1707.04131
+  - Rauber, J., Zimmermann, R., Bethge, M., and Brendel, W., "Foolbox Native: Fast adversarial attacks to benchmark the robustness of machine learning models in PyTorch, TensorFlow, and JAX," *Journal of Open Source Software*, vol. 5, no. 53, p. 2607, 2020. [Online]. Available: https://doi.org/10.21105/joss.02607
 
 We thank the authors and maintainers of these projects, as well as the authors and maintainers of the verifiers for their contributions to the robustness research community.
 

ada_verona/verification_module/attacks/foolbox_attack.py

@@ -0,0 +1,71 @@
+# Copyright 2025 ADA Reseach Group and VERONA council. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import foolbox
+from torch import Tensor, nn
+
+from ada_verona.verification_module.attacks.attack import Attack
+
+
+class FoolboxAttack(Attack):
+    """
+    A wrapper for Foolbox adversarial attacks.
+    Requires foolbox to be installed: pip install foolbox
+
+    Attributes:
+        attack_cls (class): The Foolbox attack class to use.
+        kwargs (dict): Arguments to pass to the attack constructor.
+    """
+
+    def __init__(self, attack_cls, bounds=(0, 1), **kwargs) -> None:
+        """
+        Initialize the FoolboxAttack wrapper.
+
+        Args:
+            attack_cls (class): The Foolbox attack class (e.g., foolbox.attacks.LinfPGD).
+            bounds (tuple, optional): The bounds of the input data. Defaults to (0, 1).
+            **kwargs: Arguments to be passed to the attack constructor (e.g., steps=40).
+        """
+        super().__init__()
+        self.attack_cls = attack_cls
+        self.bounds = bounds
+        self.kwargs = kwargs
+        self.name = f"FoolboxAttack ({attack_cls.__name__}, bounds={bounds}, {kwargs})"
+
+    def execute(self, model: nn.Module, data: Tensor, target: Tensor, epsilon: float) -> Tensor:
+        """
+        Execute the Foolbox attack on the given model and data.
+
+        Args:
+            model (nn.Module): The model to attack.
+            data (Tensor): The input data to perturb.
+            target (Tensor): The target labels for the data.
+            epsilon (float): The perturbation magnitude.
+
+        Returns:
+            Tensor: The perturbed data.
+        """
+        fmodel = foolbox.PyTorchModel(model, bounds=self.bounds)
+
+        attack = self.attack_cls(**self.kwargs)
+
+        if data.dim() == 3:
+            data = data.unsqueeze(0)
+
+        if target.dim() == 0:
+            target = target.unsqueeze(0)
+        _, clipped_advs, _ = attack(fmodel, data, target, epsilons=epsilon)
+
+        return clipped_advs

docs/how-to-guides.md

@@ -95,6 +95,7 @@ VERONA implements the following adversarial attack methods:
 - **Fast Gradient Sign Method (FGSM)** - [Goodfellow et al., 2015](https://arxiv.org/abs/1412.6572)
 - **Projected Gradient Descent (PGD)** - [Madry et al., 2018](https://arxiv.org/abs/1706.06083)
 - **AutoAttack** - [Croce and Hein, 2020](https://proceedings.mlr.press/v119/croce20b.html)
+- All attacks from [foolbox](https://github.com/bethgelab/foolbox/tree/master/foolbox/attacks) through the [`FoolboxAttack`](../ada_verona/verification_module/attacks/foolbox_attack.py) class
 
 ### Optional: AutoAttack Installation
 

examples/scripts/create_robustness_dist_foolbox.py

@@ -0,0 +1,84 @@
+# Copyright 2025 ADA Reseach Group and VERONA council. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import logging
+from pathlib import Path
+
+import numpy as np
+from foolbox.attacks import LinfPGD
+
+import ada_verona.util.logger as logger
+from ada_verona.database.dataset.image_file_dataset import ImageFileDataset
+from ada_verona.database.experiment_repository import ExperimentRepository
+from ada_verona.dataset_sampler.predictions_based_sampler import PredictionsBasedSampler
+from ada_verona.epsilon_value_estimator.binary_search_epsilon_value_estimator import (
+    BinarySearchEpsilonValueEstimator,
+)
+from ada_verona.verification_module.attack_estimation_module import AttackEstimationModule
+from ada_verona.verification_module.attacks.foolbox_attack import FoolboxAttack
+from ada_verona.verification_module.property_generator.one2any_property_generator import (
+    One2AnyPropertyGenerator,
+)
+
+logger.setup_logging(level=logging.INFO)
+
+experiment_name = "foolbox_pgd"
+timeout = 600
+experiment_repository_path = Path("../example_experiment/results_foolbox")
+network_folder = Path("../example_experiment/data/networks")
+image_folder = Path("../example_experiment/data/images")
+image_label_file = Path("../example_experiment/data/image_labels.csv")
+epsilon_list = np.arange(0.00, 0.4, 0.0039)
+
+dataset = ImageFileDataset(image_folder=image_folder, label_file=image_label_file)
+
+file_database = ExperimentRepository(base_path=experiment_repository_path, network_folder=network_folder)
+
+file_database.initialize_new_experiment(experiment_name)
+
+file_database.save_configuration(
+    dict(
+        experiment_name=experiment_name,
+        experiment_repository_path=str(experiment_repository_path),
+        network_folder=str(network_folder),
+        dataset=str(dataset),
+        timeout=timeout,
+        epsilon_list=[str(x) for x in epsilon_list],
+    )
+)
+
+property_generator = One2AnyPropertyGenerator()
+verifier = AttackEstimationModule(attack=FoolboxAttack(LinfPGD, bounds=(0, 1), steps=10))
+
+epsilon_value_estimator = BinarySearchEpsilonValueEstimator(epsilon_value_list=epsilon_list.copy(), verifier=verifier)
+dataset_sampler = PredictionsBasedSampler(sample_correct_predictions=True)
+
+network_list = file_database.get_network_list()
+
+print(f"Found {len(network_list)} networks.")
+
+for network in network_list:
+    print(f"Processing network: {network.name}")
+    sampled_data = dataset_sampler.sample(network, dataset)
+    print(f"Sampled {len(sampled_data)} data points.")
+
+    for i, data_point in enumerate(sampled_data):
+        print(f"Verifying data point {i}...")
+        verification_context = file_database.create_verification_context(network, data_point, property_generator)
+        epsilon_value_result = epsilon_value_estimator.compute_epsilon_value(verification_context)
+        print(f"Result: {epsilon_value_result}")
+        file_database.save_result(epsilon_value_result)
+
+print("Done.")

examples/scripts/create_robustness_dist_foolbox_cw.py

@@ -0,0 +1,90 @@
+# Copyright 2025 ADA Reseach Group and VERONA council. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import logging
+from pathlib import Path
+
+import numpy as np
+from foolbox.attacks import L2CarliniWagnerAttack
+
+import ada_verona.util.logger as logger
+from ada_verona.database.dataset.image_file_dataset import ImageFileDataset
+from ada_verona.database.experiment_repository import ExperimentRepository
+from ada_verona.dataset_sampler.predictions_based_sampler import PredictionsBasedSampler
+from ada_verona.epsilon_value_estimator.binary_search_epsilon_value_estimator import (
+    BinarySearchEpsilonValueEstimator,
+)
+from ada_verona.verification_module.attack_estimation_module import AttackEstimationModule
+from ada_verona.verification_module.attacks.foolbox_attack import FoolboxAttack
+from ada_verona.verification_module.property_generator.one2any_property_generator import (
+    One2AnyPropertyGenerator,
+)
+
+logger.setup_logging(level=logging.INFO)
+
+experiment_name = "foolbox_cw"
+timeout = 600
+experiment_repository_path = Path("../example_experiment/results_foolbox_cw")
+network_folder = Path("../example_experiment/data/networks")
+image_folder = Path("../example_experiment/data/images")
+image_label_file = Path("../example_experiment/data/image_labels.csv")
+
+epsilon_list = np.arange(0.00, 0.4, 0.0039)
+
+dataset = ImageFileDataset(image_folder=image_folder, label_file=image_label_file)
+
+file_database = ExperimentRepository(base_path=experiment_repository_path, network_folder=network_folder)
+
+file_database.initialize_new_experiment(experiment_name)
+
+file_database.save_configuration(
+    dict(
+        experiment_name=experiment_name,
+        experiment_repository_path=str(experiment_repository_path),
+        network_folder=str(network_folder),
+        dataset=str(dataset),
+        timeout=timeout,
+        epsilon_list=[str(x) for x in epsilon_list],
+    )
+)
+
+property_generator = One2AnyPropertyGenerator()
+verifier = AttackEstimationModule(attack=FoolboxAttack(L2CarliniWagnerAttack, bounds=(0, 1), steps=100))
+
+epsilon_value_estimator = BinarySearchEpsilonValueEstimator(epsilon_value_list=epsilon_list.copy(), verifier=verifier)
+dataset_sampler = PredictionsBasedSampler(sample_correct_predictions=True)
+
+network_list = file_database.get_network_list()
+
+print(f"Found {len(network_list)} networks.")
+
+for network in network_list:
+    print(f"Processing network: {network.name}")
+    sampled_data = dataset_sampler.sample(network, dataset)
+    print(f"Sampled {len(sampled_data)} data points.")
+
+    for i, data_point in enumerate(sampled_data):
+        if i >= 1:
+            break
+
+        print(f"Verifying data point {i}...")
+        verification_context = file_database.create_verification_context(network, data_point, property_generator)
+
+        epsilon_value_result = epsilon_value_estimator.compute_epsilon_value(verification_context)
+
+        print(f"Result: {epsilon_value_result}")
+        file_database.save_result(epsilon_value_result)
+
+print("Done.")

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
     "onnx>=1.14.0",
     "onnxruntime>=1.14.1",
     "onnx2torch>=1.5.14",
+    "foolbox>=3.3.4",
     "pandas>=2.0.1",
     "PyYAML>=6.0.1",
     "result>=0.9.0",

tests/test_verification_module/attacks/conftest.py

@@ -13,12 +13,14 @@
 # limitations under the License.
 # ==============================================================================
 
+import foolbox as fb
 import pytest
 import torch
 from torch import nn
 
 from ada_verona.verification_module.attacks.auto_attack_wrapper import AutoAttackWrapper
 from ada_verona.verification_module.attacks.fgsm_attack import FGSMAttack
+from ada_verona.verification_module.attacks.foolbox_attack import FoolboxAttack
 from ada_verona.verification_module.attacks.pgd_attack import PGDAttack
 
 
@@ -34,22 +36,32 @@ def forward(self, x):
 
     return SimpleModel()
 
+
 @pytest.fixture
 def data():
     return torch.randn(1, 10)
 
+
 @pytest.fixture
 def target():
     return torch.tensor([1])
 
+
 @pytest.fixture
 def attack_wrapper():
     return AutoAttackWrapper(device="cpu", norm="Linf", version="standard", verbose=False)
 
+
 @pytest.fixture
 def pgd_attack():
-     return PGDAttack(number_iterations=10, step_size=0.01, randomise=True)
+    return PGDAttack(number_iterations=10, step_size=0.01, randomise=True)
+
 
 @pytest.fixture
 def fgsm_attack():
-    return FGSMAttack()
+    return FGSMAttack()
+
+
+@pytest.fixture
+def foolbox_attack():
+    return FoolboxAttack(attack_cls=fb.attacks.LinfFastGradientAttack)

tests/test_verification_module/attacks/test_foolbox_attack.py

@@ -0,0 +1,24 @@
+# Copyright 2025 ADA Reseach Group and VERONA council. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import torch
+
+
+def test_foolbox_attack_execute(foolbox_attack, model, data, target):
+    epsilon = 0.1
+    perturbed_data = foolbox_attack.execute(model, data, target, epsilon)
+    assert isinstance(perturbed_data, torch.Tensor)
+    assert perturbed_data.shape == data.shape
+    assert torch.all(perturbed_data >= 0) and torch.all(perturbed_data <= 1)