From 8a8f07d678b62b8b51fee2092a3264e2a674c278 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Mon, 25 Sep 2023 12:09:34 -0700 Subject: [PATCH 01/17] Composer-Functions. --- mart/attack/composer.py | 79 +++++++++++++++---- mart/configs/attack/adversary.yaml | 2 +- .../attack/classification_fgsm_linf.yaml | 2 +- .../attack/classification_pgd_linf.yaml | 2 +- mart/configs/attack/composer/additive.yaml | 1 - mart/configs/attack/composer/default.yaml | 2 + .../attack/composer/functions/additive.yaml | 3 + .../attack/composer/functions/mask.yaml | 3 + .../attack/composer/functions/overlay.yaml | 3 + .../attack/composer/mask_additive.yaml | 1 - mart/configs/attack/composer/overlay.yaml | 1 - .../object_detection_mask_adversary.yaml | 2 +- ...bject_detection_mask_adversary_missed.yaml | 2 +- 13 files changed, 79 insertions(+), 24 deletions(-) delete mode 100644 mart/configs/attack/composer/additive.yaml create mode 100644 mart/configs/attack/composer/default.yaml create mode 100644 mart/configs/attack/composer/functions/additive.yaml create mode 100644 mart/configs/attack/composer/functions/mask.yaml create mode 100644 mart/configs/attack/composer/functions/overlay.yaml delete mode 100644 mart/configs/attack/composer/mask_additive.yaml delete mode 100644 mart/configs/attack/composer/overlay.yaml diff --git a/mart/attack/composer.py b/mart/attack/composer.py index 6b40950a..c512489e 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -7,12 +7,32 @@ from __future__ import annotations import abc +from collections import OrderedDict from typing import Any, Iterable import torch -class Composer(abc.ABC): +class Function(torch.nn.Module): + def __init__(self, *args, order=0, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.order = order + + @abc.abstractmethod + def forward(self, perturbation, input, target) -> None: + """Returns the modified perturbation, modified input and target, so we can chain Functions + in a Composer.""" + pass + + +class Composer: + def __init__(self, functions: dict[str, Function]) -> None: + # Sort functions by function.order and the name. + self.functions_dict = OrderedDict( + sorted(functions.items(), key=lambda name_fn: (name_fn[1].order, name_fn[0])) + ) + self.functions = list(self.functions_dict.values()) + def __call__( self, perturbation: torch.Tensor | Iterable[torch.Tensor], @@ -38,7 +58,6 @@ def __call__( else: raise NotImplementedError - @abc.abstractmethod def compose( self, perturbation: torch.Tensor, @@ -46,35 +65,63 @@ def compose( input: torch.Tensor, target: torch.Tensor | dict[str, Any], ) -> torch.Tensor: - raise NotImplementedError + for function in self.functions: + perturbation, input, target = function(perturbation, input, target) + + # Return the composed input. + return input -class Additive(Composer): +class Additive(Function): """We assume an adversary adds perturbation to the input.""" - def compose(self, perturbation, *, input, target): - return input + perturbation + def forward(self, perturbation, input, target): + input = input + perturbation + return perturbation, input, target -class Overlay(Composer): +class Mask(Function): + def __init__(self, key="perturbable_mask"): + self.key = key + + def forward(self, perturbation, input, target): + mask = target[self.key] + perturbation = perturbation * mask + return perturbation, input, target + + +class Overlay(Function): """We assume an adversary overlays a patch to the input.""" - def compose(self, perturbation, *, input, target): + def __init__(self, key="perturbable_mask"): + self.key = key + + def forward(self, perturbation, input, target): # True is mutable, False is immutable. - mask = target["perturbable_mask"] + mask = target[self.key] # Convert mask to a Tensor with same torch.dtype and torch.device as input, # because some data modules (e.g. Armory) gives binary mask. mask = mask.to(input) - return input * (1 - mask) + perturbation * mask + perturbation = perturbation * mask + + input = input * (1 - mask) + perturbation + return perturbation, input, target + +class MaskAdditive(Function): + """We assume an adversary adds masked perturbation to the input. -class MaskAdditive(Composer): - """We assume an adversary adds masked perturbation to the input.""" + Backward compatible. Could be deleted later. + """ - def compose(self, perturbation, *, input, target): - mask = target["perturbable_mask"] - masked_perturbation = perturbation * mask + def __init__(self, key="perturbable_mask"): + function_mask = Mask(key=key) + function_additive = Additive() + self.functions = [function_mask, function_additive] - return input + masked_perturbation + def compose(self, perturbation, input, target): + for function in self.functions: + perturbation, input, target = function(perturbation, input, target) + return perturbation, input, target diff --git a/mart/configs/attack/adversary.yaml b/mart/configs/attack/adversary.yaml index bbf52433..5f65f99d 100644 --- a/mart/configs/attack/adversary.yaml +++ b/mart/configs/attack/adversary.yaml @@ -1,10 +1,10 @@ defaults: - /callbacks@callbacks: [progress_bar] + - composer: default _target_: mart.attack.Adversary _convert_: all perturber: ??? -composer: ??? optimizer: maximize: True gain: ??? diff --git a/mart/configs/attack/classification_fgsm_linf.yaml b/mart/configs/attack/classification_fgsm_linf.yaml index 45465429..74c9b959 100644 --- a/mart/configs/attack/classification_fgsm_linf.yaml +++ b/mart/configs/attack/classification_fgsm_linf.yaml @@ -2,7 +2,7 @@ defaults: - adversary - fgm - linf - - composer: additive + - composer/functions: additive - gradient_modifier: sign - gain: cross_entropy - objective: misclassification diff --git a/mart/configs/attack/classification_pgd_linf.yaml b/mart/configs/attack/classification_pgd_linf.yaml index b2e8ddfd..fec19029 100644 --- a/mart/configs/attack/classification_pgd_linf.yaml +++ b/mart/configs/attack/classification_pgd_linf.yaml @@ -2,7 +2,7 @@ defaults: - adversary - pgd - linf - - composer: additive + - composer/functions: additive - gradient_modifier: sign - gain: cross_entropy - objective: misclassification diff --git a/mart/configs/attack/composer/additive.yaml b/mart/configs/attack/composer/additive.yaml deleted file mode 100644 index c12f939e..00000000 --- a/mart/configs/attack/composer/additive.yaml +++ /dev/null @@ -1 +0,0 @@ -_target_: mart.attack.composer.Additive diff --git a/mart/configs/attack/composer/default.yaml b/mart/configs/attack/composer/default.yaml new file mode 100644 index 00000000..91a25390 --- /dev/null +++ b/mart/configs/attack/composer/default.yaml @@ -0,0 +1,2 @@ +_target_: mart.attack.Composer +functions: ??? diff --git a/mart/configs/attack/composer/functions/additive.yaml b/mart/configs/attack/composer/functions/additive.yaml new file mode 100644 index 00000000..e8932644 --- /dev/null +++ b/mart/configs/attack/composer/functions/additive.yaml @@ -0,0 +1,3 @@ +additive: + _target_: mart.attack.composer.Additive + order: 0 diff --git a/mart/configs/attack/composer/functions/mask.yaml b/mart/configs/attack/composer/functions/mask.yaml new file mode 100644 index 00000000..98b5125e --- /dev/null +++ b/mart/configs/attack/composer/functions/mask.yaml @@ -0,0 +1,3 @@ +mask: + _target_: mart.attack.composer.Mask + order: 0 diff --git a/mart/configs/attack/composer/functions/overlay.yaml b/mart/configs/attack/composer/functions/overlay.yaml new file mode 100644 index 00000000..5ff068e9 --- /dev/null +++ b/mart/configs/attack/composer/functions/overlay.yaml @@ -0,0 +1,3 @@ +overlay: + _target_: mart.attack.composer.Overlay + order: 0 diff --git a/mart/configs/attack/composer/mask_additive.yaml b/mart/configs/attack/composer/mask_additive.yaml deleted file mode 100644 index 4bca36f8..00000000 --- a/mart/configs/attack/composer/mask_additive.yaml +++ /dev/null @@ -1 +0,0 @@ -_target_: mart.attack.composer.MaskAdditive diff --git a/mart/configs/attack/composer/overlay.yaml b/mart/configs/attack/composer/overlay.yaml deleted file mode 100644 index 469f7245..00000000 --- a/mart/configs/attack/composer/overlay.yaml +++ /dev/null @@ -1 +0,0 @@ -_target_: mart.attack.composer.Overlay diff --git a/mart/configs/attack/object_detection_mask_adversary.yaml b/mart/configs/attack/object_detection_mask_adversary.yaml index 0e42cb61..bc88ba3d 100644 --- a/mart/configs/attack/object_detection_mask_adversary.yaml +++ b/mart/configs/attack/object_detection_mask_adversary.yaml @@ -3,7 +3,7 @@ defaults: - gradient_ascent - mask - perturber/initializer: constant - - composer: overlay + - composer/functions: overlay - gradient_modifier: sign - gain: rcnn_training_loss - objective: zero_ap diff --git a/mart/configs/attack/object_detection_mask_adversary_missed.yaml b/mart/configs/attack/object_detection_mask_adversary_missed.yaml index 4f5fc039..6be9ec8b 100644 --- a/mart/configs/attack/object_detection_mask_adversary_missed.yaml +++ b/mart/configs/attack/object_detection_mask_adversary_missed.yaml @@ -3,7 +3,7 @@ defaults: - gradient_ascent - mask - perturber/initializer: constant - - composer: overlay + - composer/functions: overlay - gradient_modifier: sign - gain: rcnn_class_background - objective: object_detection_missed From 86c8177ffb582252afc0bda48d7f0fc373c2179a Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Mon, 25 Sep 2023 12:15:07 -0700 Subject: [PATCH 02/17] Get both key and order arguments. --- mart/attack/composer.py | 6 ++++-- mart/configs/attack/composer/functions/mask.yaml | 1 + mart/configs/attack/composer/functions/overlay.yaml | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mart/attack/composer.py b/mart/attack/composer.py index c512489e..ef747f45 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -81,7 +81,8 @@ def forward(self, perturbation, input, target): class Mask(Function): - def __init__(self, key="perturbable_mask"): + def __init__(self, *args, key="perturbable_mask", **kwargs): + super().__init__(*args, **kwargs) self.key = key def forward(self, perturbation, input, target): @@ -93,7 +94,8 @@ def forward(self, perturbation, input, target): class Overlay(Function): """We assume an adversary overlays a patch to the input.""" - def __init__(self, key="perturbable_mask"): + def __init__(self, *args, key="perturbable_mask", **kwargs): + super().__init__(*args, **kwargs) self.key = key def forward(self, perturbation, input, target): diff --git a/mart/configs/attack/composer/functions/mask.yaml b/mart/configs/attack/composer/functions/mask.yaml index 98b5125e..04cefaf8 100644 --- a/mart/configs/attack/composer/functions/mask.yaml +++ b/mart/configs/attack/composer/functions/mask.yaml @@ -1,3 +1,4 @@ mask: _target_: mart.attack.composer.Mask + key: perturbable_mask order: 0 diff --git a/mart/configs/attack/composer/functions/overlay.yaml b/mart/configs/attack/composer/functions/overlay.yaml index 5ff068e9..d6137ad1 100644 --- a/mart/configs/attack/composer/functions/overlay.yaml +++ b/mart/configs/attack/composer/functions/overlay.yaml @@ -1,3 +1,4 @@ overlay: _target_: mart.attack.composer.Overlay + key: perturbable_mask order: 0 From d583928670754fa5dc32b1a43cf6b5d3ba367627 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Mon, 25 Sep 2023 12:16:21 -0700 Subject: [PATCH 03/17] Implement mask-additive in config. --- mart/attack/composer.py | 17 ----------------- mart/configs/attack/composer/mask_additive.yaml | 9 +++++++++ 2 files changed, 9 insertions(+), 17 deletions(-) create mode 100644 mart/configs/attack/composer/mask_additive.yaml diff --git a/mart/attack/composer.py b/mart/attack/composer.py index ef747f45..458b23e1 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -110,20 +110,3 @@ def forward(self, perturbation, input, target): input = input * (1 - mask) + perturbation return perturbation, input, target - - -class MaskAdditive(Function): - """We assume an adversary adds masked perturbation to the input. - - Backward compatible. Could be deleted later. - """ - - def __init__(self, key="perturbable_mask"): - function_mask = Mask(key=key) - function_additive = Additive() - self.functions = [function_mask, function_additive] - - def compose(self, perturbation, input, target): - for function in self.functions: - perturbation, input, target = function(perturbation, input, target) - return perturbation, input, target diff --git a/mart/configs/attack/composer/mask_additive.yaml b/mart/configs/attack/composer/mask_additive.yaml new file mode 100644 index 00000000..bb2bde70 --- /dev/null +++ b/mart/configs/attack/composer/mask_additive.yaml @@ -0,0 +1,9 @@ +defaults: + - default + - functions: [mask, additive] + +functions: + mask: + order: 0 + additive: + order: 1 From 16a5ef29a63fb8ac2316b20b548010a1b0510554 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Mon, 25 Sep 2023 15:05:18 -0700 Subject: [PATCH 04/17] Comment. --- mart/attack/composer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mart/attack/composer.py b/mart/attack/composer.py index 458b23e1..a54e8db2 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -15,6 +15,11 @@ class Function(torch.nn.Module): def __init__(self, *args, order=0, **kwargs) -> None: + """A stackable function for Composer. + + Args: + order (int, optional): The priority number. A smaller number makes a function run earlier than others in a sequence. Defaults to 0. + """ super().__init__(*args, **kwargs) self.order = order From 0dfb93b293e146d602c936397289e089be311103 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Mon, 25 Sep 2023 15:09:01 -0700 Subject: [PATCH 05/17] Fix test. --- tests/test_composer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_composer.py b/tests/test_composer.py index 9fc15cf8..04851c5f 100644 --- a/tests/test_composer.py +++ b/tests/test_composer.py @@ -6,11 +6,11 @@ import torch -from mart.attack.composer import Additive, MaskAdditive, Overlay +from mart.attack.composer import Additive, Composer, Mask, Overlay def test_additive_composer_forward(input_data, target_data, perturbation): - composer = Additive() + composer = Composer(functions={"additive": Additive()}) output = composer(perturbation, input=input_data, target=target_data) expected_output = input_data + perturbation @@ -18,7 +18,7 @@ def test_additive_composer_forward(input_data, target_data, perturbation): def test_overlay_composer_forward(input_data, target_data, perturbation): - composer = Overlay() + composer = Composer(functions={"overlay": Overlay()}) output = composer(perturbation, input=input_data, target=target_data) mask = target_data["perturbable_mask"] @@ -33,6 +33,6 @@ def test_mask_additive_composer_forward(): target = {"perturbable_mask": torch.eye(2)} expected_output = torch.eye(2) - composer = MaskAdditive() + composer = Composer(functions={"mask": Mask(order=0), "additive": Additive(order=1)}) output = composer(perturbation, input=input, target=target) torch.testing.assert_close(output, expected_output, equal_nan=True) From 00cb32f55e2eedb958fc9d227893e9bd9ca89c53 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Mon, 25 Sep 2023 15:14:30 -0700 Subject: [PATCH 06/17] Fix tests. --- tests/test_adversary.py | 44 ++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/tests/test_adversary.py b/tests/test_adversary.py index 2113d3f6..b68a1052 100644 --- a/tests/test_adversary.py +++ b/tests/test_adversary.py @@ -19,7 +19,9 @@ def test_with_model(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) gain = Mock() enforcer = Mock() attacker = Mock(max_epochs=0, limit_train_batches=1, fit_loop=Mock(max_epochs=0)) @@ -52,7 +54,9 @@ def test_with_model(input_data, target_data, perturbation): def test_hidden_params(): initializer = Mock() - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) projector = Mock() perturber = Perturber(initializer=initializer, projector=projector) @@ -81,7 +85,9 @@ def test_hidden_params(): def test_hidden_params_after_forward(input_data, target_data, perturbation): initializer = Mock() - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) projector = Mock() perturber = Perturber(initializer=initializer, projector=projector) @@ -115,7 +121,9 @@ def test_hidden_params_after_forward(input_data, target_data, perturbation): def test_loading_perturbation_from_state_dict(): initializer = Mock() - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) projector = Mock() perturber = Perturber(initializer=initializer, projector=projector) @@ -144,7 +152,9 @@ def test_loading_perturbation_from_state_dict(): def test_perturbation(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) gain = Mock() enforcer = Mock() attacker = Mock(max_epochs=0, limit_train_batches=1, fit_loop=Mock(max_epochs=0)) @@ -175,7 +185,9 @@ def test_perturbation(input_data, target_data, perturbation): def test_forward_with_model(input_data, target_data): - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) enforcer = Mock() optimizer = partial(SGD, lr=1.0, maximize=True) @@ -219,7 +231,9 @@ def model(input, target): def test_configure_optimizers(): perturber = Mock() - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) optimizer = Mock(spec=mart.optim.OptimizerFactory) gain = Mock() @@ -238,7 +252,9 @@ def test_configure_optimizers(): def test_training_step(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) optimizer = Mock(spec=mart.optim.OptimizerFactory) gain = Mock(return_value=torch.tensor(1337)) model = Mock(spec="__call__", return_value={}) @@ -259,7 +275,9 @@ def test_training_step(input_data, target_data, perturbation): def test_training_step_with_many_gain(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) optimizer = Mock(spec=mart.optim.OptimizerFactory) gain = Mock(return_value=torch.tensor([1234, 5678])) model = Mock(spec="__call__", return_value={}) @@ -279,7 +297,9 @@ def test_training_step_with_many_gain(input_data, target_data, perturbation): def test_training_step_with_objective(input_data, target_data, perturbation): perturber = Mock(spec=Perturber, return_value=perturbation) - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) optimizer = Mock(spec=mart.optim.OptimizerFactory) gain = Mock(return_value=torch.tensor([1234, 5678])) # The model has no attack_step() or training_step(). @@ -304,7 +324,9 @@ def test_training_step_with_objective(input_data, target_data, perturbation): def test_configure_gradient_clipping(): perturber = Mock() - composer = mart.attack.composer.Additive() + composer = mart.attack.composer.Composer( + functions={"additive": mart.attack.composer.Additive()} + ) optimizer = Mock( spec=mart.optim.OptimizerFactory, param_groups=[{"params": Mock()}, {"params": Mock()}] ) From 11c7a47d67d043a1782c7d08031b6dae55727b42 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Mon, 25 Sep 2023 21:08:10 -0700 Subject: [PATCH 07/17] Add composer functions for rectangle patch. --- mart/attack/composer.py | 112 ++++++++++++++++++ .../attack/composer/functions/fake_clamp.yaml | 5 + .../attack/composer/functions/rect_crop.yaml | 4 + .../attack/composer/functions/rect_pad.yaml | 5 + .../functions/rect_perspective_transform.yaml | 5 + .../composer/rectangle_patch_additive.yaml | 25 ++++ .../composer/rectangle_patch_overlay.yaml | 16 +++ tests/test_composer.py | 80 ++++++++++++- 8 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 mart/configs/attack/composer/functions/fake_clamp.yaml create mode 100644 mart/configs/attack/composer/functions/rect_crop.yaml create mode 100644 mart/configs/attack/composer/functions/rect_pad.yaml create mode 100644 mart/configs/attack/composer/functions/rect_perspective_transform.yaml create mode 100644 mart/configs/attack/composer/rectangle_patch_additive.yaml create mode 100644 mart/configs/attack/composer/rectangle_patch_overlay.yaml diff --git a/mart/attack/composer.py b/mart/attack/composer.py index a54e8db2..d98903b0 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -11,6 +11,7 @@ from typing import Any, Iterable import torch +from torchvision.transforms import functional as F class Function(torch.nn.Module): @@ -115,3 +116,114 @@ def forward(self, perturbation, input, target): input = input * (1 - mask) + perturbation return perturbation, input, target + + +class RectangleCrop(Function): + def __init__(self, *args, coords_key="patch_coords", **kwargs): + super().__init__(*args, **kwargs) + self.coords_key = coords_key + + def get_smallest_rectangle_shape(self, input, patch_coords): + """Get a smallest rectangle that covers the whole patch.""" + coords = patch_coords + leading_dims = list(input.shape[:-2]) + width = coords[:, 0].max() - coords[:, 0].min() + height = coords[:, 1].max() - coords[:, 1].min() + shape = list(leading_dims) + [height, width] + return shape + + def slice_rectangle(self, perturbation, height_patch, width_patch): + """Slice a rectangle from top-left of the perturbation.""" + height_patch_index = torch.tensor(range(height_patch), device=perturbation.device) + width_patch_index = torch.tensor(range(width_patch), device=perturbation.device) + perturbation_patch = perturbation.index_select(-2, height_patch_index).index_select( + -1, width_patch_index + ) + return perturbation_patch + + def forward(self, perturbation, input, target): + coords = target[self.coords_key] + # TODO: Make composers stackable to reuse some Composer. + # The perturbation variable has the same shape as input. + # We slice a small rectangle from top-left of the perturbation variable to compose the patch. + rectangle_shape = self.get_smallest_rectangle_shape(input, coords) + # Assume perturbation is in shape of [N]CHW + height_patch, width_patch = rectangle_shape[-2:] + rectangle_patch = self.slice_rectangle(perturbation, height_patch, width_patch) + return rectangle_patch, input, target + + +class RectanglePad(Function): + def __init__(self, *args, coords_key="patch_coords", rect_coords_key="rect_coords", **kwargs): + super().__init__(*args, **kwargs) + self.coords_key = coords_key + self.rect_coords_key = rect_coords_key + + def forward(self, perturbation_patch, input, target): + coords = target[self.coords_key] + height, width = input.shape[-2:] + # Pad rectangle to the same size of input, so that it is almost aligned with the patch. + height_patch, width_patch = perturbation_patch.shape[-2:] + pad_left = min(coords[0, 0], coords[3, 0]) + pad_top = min(coords[0, 1], coords[1, 1]) + pad_right = width - width_patch - pad_left + pad_bottom = height - height_patch - pad_top + + perturbation_padded = F.pad( + img=perturbation_patch, + padding=[pad_left, pad_top, pad_right, pad_bottom], + fill=0, + padding_mode="constant", + ) + + # Save coords of four corners of the rectangle for later transform. + top_left = [pad_left, pad_top] + top_right = [width - pad_right, pad_top] + bottom_right = [width - pad_right, height - pad_bottom] + bottom_left = [pad_left, height - pad_bottom] + target[self.rect_coords_key] = [top_left, top_right, bottom_right, bottom_left] + + return perturbation_padded, input, target + + +class RectanglePerspectiveTransform(Function): + def __init__(self, *args, coords_key="patch_coords", rect_coords_key="rect_coords", **kwargs): + super().__init__(*args, **kwargs) + self.coords_key = coords_key + self.rect_coords_key = rect_coords_key + + def forward(self, perturbation_rect, input, target): + coords = target[self.coords_key] + # Perspective transformation: rectangle -> coords. + # Fetch four corners of the rectangle. + startpoints = target[self.rect_coords_key] + endpoints = coords + # TODO: Make interpolation configurable. + perturbation_coords = F.perspective( + img=perturbation_rect, + startpoints=startpoints, + endpoints=endpoints, + interpolation=F.InterpolationMode.BILINEAR, + fill=0, + ) + return perturbation_coords, input, target + + +class FakeClamp(Function): + """A Clamp operation that preserves gradients.""" + + def __init__(self, *args, min_val, max_val, **kwargs): + super().__init__(*args, **kwargs) + self.min_val = min_val + self.max_val = max_val + + @staticmethod + def fake_clamp(x, *, min_val, max_val): + with torch.no_grad(): + x_clamped = x.clamp(min_val, max_val) + diff = x_clamped - x + return x + diff + + def forward(self, perturbation, input, target): + input = self.fake_clamp(input, min_val=self.min_val, max_val=self.max_val) + return perturbation, input, target diff --git a/mart/configs/attack/composer/functions/fake_clamp.yaml b/mart/configs/attack/composer/functions/fake_clamp.yaml new file mode 100644 index 00000000..50ef4fbe --- /dev/null +++ b/mart/configs/attack/composer/functions/fake_clamp.yaml @@ -0,0 +1,5 @@ +fake_clamp: + _target_: mart.attack.composer.FakeClamp + order: 0 + min_val: 0 + max_val: 255 diff --git a/mart/configs/attack/composer/functions/rect_crop.yaml b/mart/configs/attack/composer/functions/rect_crop.yaml new file mode 100644 index 00000000..011cd6ff --- /dev/null +++ b/mart/configs/attack/composer/functions/rect_crop.yaml @@ -0,0 +1,4 @@ +rect_crop: + _target_: mart.attack.composer.RectangleCrop + coords_key: patch_coords + order: 0 diff --git a/mart/configs/attack/composer/functions/rect_pad.yaml b/mart/configs/attack/composer/functions/rect_pad.yaml new file mode 100644 index 00000000..384a71fd --- /dev/null +++ b/mart/configs/attack/composer/functions/rect_pad.yaml @@ -0,0 +1,5 @@ +rect_pad: + _target_: mart.attack.composer.RectanglePad + coords_key: patch_coords + rect_coords_key: rect_coords + order: 0 diff --git a/mart/configs/attack/composer/functions/rect_perspective_transform.yaml b/mart/configs/attack/composer/functions/rect_perspective_transform.yaml new file mode 100644 index 00000000..87eac864 --- /dev/null +++ b/mart/configs/attack/composer/functions/rect_perspective_transform.yaml @@ -0,0 +1,5 @@ +rect_perspective_transform: + _target_: mart.attack.composer.RectanglePerspectiveTransform + order: 0 + coords_key: patch_coords + rect_coords_key: rect_coords diff --git a/mart/configs/attack/composer/rectangle_patch_additive.yaml b/mart/configs/attack/composer/rectangle_patch_additive.yaml new file mode 100644 index 00000000..0f391299 --- /dev/null +++ b/mart/configs/attack/composer/rectangle_patch_additive.yaml @@ -0,0 +1,25 @@ +defaults: + - default + - functions: + [ + rect_crop, + rect_pad, + rect_perspective_transform, + mask, + additive, + fake_clamp, + ] + +functions: + rect_crop: + order: 0 + rect_pad: + order: 1 + rect_perspective_transform: + order: 2 + mask: + order: 3 + additive: + order: 4 + fake_clamp: + order: 5 diff --git a/mart/configs/attack/composer/rectangle_patch_overlay.yaml b/mart/configs/attack/composer/rectangle_patch_overlay.yaml new file mode 100644 index 00000000..28a3946a --- /dev/null +++ b/mart/configs/attack/composer/rectangle_patch_overlay.yaml @@ -0,0 +1,16 @@ +defaults: + - default + - functions: + [rect_crop, rect_pad, rect_perspective_transform, overlay, fake_clamp] + +functions: + rect_crop: + order: 0 + rect_pad: + order: 1 + rect_perspective_transform: + order: 2 + overlay: + order: 3 + fake_clamp: + order: 4 diff --git a/tests/test_composer.py b/tests/test_composer.py index 04851c5f..93db2abd 100644 --- a/tests/test_composer.py +++ b/tests/test_composer.py @@ -6,7 +6,15 @@ import torch -from mart.attack.composer import Additive, Composer, Mask, Overlay +from mart.attack.composer import ( + Additive, + Composer, + Mask, + Overlay, + RectangleCrop, + RectanglePad, + RectanglePerspectiveTransform, +) def test_additive_composer_forward(input_data, target_data, perturbation): @@ -36,3 +44,73 @@ def test_mask_additive_composer_forward(): composer = Composer(functions={"mask": Mask(order=0), "additive": Additive(order=1)}) output = composer(perturbation, input=input, target=target) torch.testing.assert_close(output, expected_output, equal_nan=True) + + +def test_rect_crop(): + key = "patch_coords" + input = torch.zeros((3, 10, 10)) + perturbation = torch.ones_like(input) + fn = RectangleCrop(coords_key=key) + + # FIXME: four corner points (width, height) of a patch in the order of top-left, top-right, bottom-right, bottom-left. + # A simple square patch. + patch_coords = torch.tensor(((0, 0), (5, 0), (5, 5), (5, 0))) + target = {key: patch_coords} + + rect_patch, _input, _target = fn(perturbation, input, target) + assert torch.equal(input, _input) + assert target == _target + assert rect_patch.shape == (3, 5, 5) + + # A skew patch. + patch_coords = torch.tensor(((1, 1), (5, 2), (7, 8), (3, 9))) + target = {key: patch_coords} + + rect_patch, _input, _target = fn(perturbation, input, target) + assert torch.equal(input, _input) + assert target == _target + assert rect_patch.shape == (3, 8, 6) + + +def test_rect_pad(): + coords_key = "patch_coords" + rect_coords_key = "rect_coords" + + rect_patch = torch.ones(3, 5, 5) + patch_coords = torch.tensor(((0, 0), (5, 0), (5, 5), (5, 0))) + + input = torch.zeros((3, 10, 10)) + target = {coords_key: patch_coords} + + fn = RectanglePad(coords_key=coords_key, rect_coords_key=rect_coords_key) + pert_padded, _input, _target = fn(rect_patch, input, target) + + pert_padded_expected = torch.zeros_like(input) + pert_padded_expected[:, :5, :5] = 1 + + assert torch.equal(pert_padded_expected, pert_padded) + + rect_coords_expected = [[0, 0], [5, 0], [5, 5], [0, 5]] + assert _target[rect_coords_key] == rect_coords_expected + + +def test_rect_perspective_transform(): + coords_key = "patch_coords" + rect_coords_key = "rect_coords" + + rect_coords = [[0, 0], [5, 0], [5, 5], [0, 5]] + # Move from top left to bottom right. + patch_coords = torch.tensor(((5, 5), (10, 5), (10, 10), (5, 10))) + target = {coords_key: patch_coords, rect_coords_key: rect_coords} + + input = torch.zeros((3, 10, 10)) + + pert_padded = torch.zeros_like(input) + pert_padded[:, :5, :5] = 1 + + fn = RectanglePerspectiveTransform(coords_key=coords_key, rect_coords_key=rect_coords_key) + pert_coords, _input, _target = fn(pert_padded, input, target) + pert_coords_expected = torch.zeros_like(input) + pert_coords_expected[:, 5:, 5:] = 1 + # rounding numeric error from the perspective transformation. + assert torch.equal(pert_coords.round(), pert_coords_expected) From 4711fd34a15c94b53c5c5e1a6b78ac002cf2ffc1 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Wed, 27 Sep 2023 10:40:10 -0700 Subject: [PATCH 08/17] Simplify names. --- .../{rectangle_patch_additive.yaml => rect_patch_additive.yaml} | 0 .../{rectangle_patch_overlay.yaml => rect_patch_overlay.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename mart/configs/attack/composer/{rectangle_patch_additive.yaml => rect_patch_additive.yaml} (100%) rename mart/configs/attack/composer/{rectangle_patch_overlay.yaml => rect_patch_overlay.yaml} (100%) diff --git a/mart/configs/attack/composer/rectangle_patch_additive.yaml b/mart/configs/attack/composer/rect_patch_additive.yaml similarity index 100% rename from mart/configs/attack/composer/rectangle_patch_additive.yaml rename to mart/configs/attack/composer/rect_patch_additive.yaml diff --git a/mart/configs/attack/composer/rectangle_patch_overlay.yaml b/mart/configs/attack/composer/rect_patch_overlay.yaml similarity index 100% rename from mart/configs/attack/composer/rectangle_patch_overlay.yaml rename to mart/configs/attack/composer/rect_patch_overlay.yaml From 957900d696462d86626be18db96556484e28e896 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 12:40:30 -0700 Subject: [PATCH 09/17] Add perturbation image. --- mart/attack/composer.py | 32 ++++++++++++++++++- .../attack/composer/functions/pert_image.yaml | 4 +++ .../attack/composer/rect_patch_additive.yaml | 13 +++++--- .../attack/composer/rect_patch_overlay.yaml | 19 ++++++++--- 4 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 mart/configs/attack/composer/functions/pert_image.yaml diff --git a/mart/attack/composer.py b/mart/attack/composer.py index 1247b6b3..a0e31367 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -11,7 +11,12 @@ from typing import Any, Iterable import torch -from torchvision.transforms import functional as F +import torchvision +import torchvision.transforms.functional as F + +from mart.utils import pylogger + +logger = pylogger.get_pylogger(__name__) class Function(torch.nn.Module): @@ -229,3 +234,28 @@ def fake_clamp(x, *, min_val, max_val): def forward(self, perturbation, input, target): input = self.fake_clamp(input, min_val=self.min_val, max_val=self.max_val) return perturbation, input, target + + +class PerturbationImage(Function): + """Add an image to perturbation if specified.""" + + def __init__(self, *args, path: str | None = None, scale: int = 1, **kwargs): + super().__init__(*args, **kwargs) + + self.image = None + if path is not None: + # This is uint8 [0,255]. + self.image = torchvision.io.read_image(path, torchvision.io.ImageReadMode.RGB) + # We shouldn't need scale as we use canonical input format. + self.image = self.image / scale + + def forward(self, perturbation, input, target): + if self.image is not None: + image = self.image + + if image.shape != perturbation.shape: + logger.info(f"Resizing image from {image.shape} to {perturbation.shape}...") + image = F.resize(image, perturbation.shape[1:]) + + perturbation = perturbation + image + return perturbation, input, target diff --git a/mart/configs/attack/composer/functions/pert_image.yaml b/mart/configs/attack/composer/functions/pert_image.yaml new file mode 100644 index 00000000..848a5d5d --- /dev/null +++ b/mart/configs/attack/composer/functions/pert_image.yaml @@ -0,0 +1,4 @@ +pert_image: + _target_: mart.attack.composer.PerturbationImage + path: null + order: 0 diff --git a/mart/configs/attack/composer/rect_patch_additive.yaml b/mart/configs/attack/composer/rect_patch_additive.yaml index 0f391299..5a0247ce 100644 --- a/mart/configs/attack/composer/rect_patch_additive.yaml +++ b/mart/configs/attack/composer/rect_patch_additive.yaml @@ -3,6 +3,7 @@ defaults: - functions: [ rect_crop, + pert_image, rect_pad, rect_perspective_transform, mask, @@ -13,13 +14,15 @@ defaults: functions: rect_crop: order: 0 - rect_pad: + pert_image: order: 1 - rect_perspective_transform: + rect_pad: order: 2 - mask: + rect_perspective_transform: order: 3 - additive: + mask: order: 4 - fake_clamp: + additive: order: 5 + fake_clamp: + order: 6 diff --git a/mart/configs/attack/composer/rect_patch_overlay.yaml b/mart/configs/attack/composer/rect_patch_overlay.yaml index 28a3946a..8e1ab67a 100644 --- a/mart/configs/attack/composer/rect_patch_overlay.yaml +++ b/mart/configs/attack/composer/rect_patch_overlay.yaml @@ -1,16 +1,25 @@ defaults: - default - functions: - [rect_crop, rect_pad, rect_perspective_transform, overlay, fake_clamp] + [ + rect_crop, + pert_image, + rect_pad, + rect_perspective_transform, + overlay, + fake_clamp, + ] functions: rect_crop: order: 0 - rect_pad: + pert_image: order: 1 - rect_perspective_transform: + rect_pad: order: 2 - overlay: + rect_perspective_transform: order: 3 - fake_clamp: + overlay: order: 4 + fake_clamp: + order: 5 From 7603cd3a44a90cc967863c95496df124890fddc4 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 12:46:21 -0700 Subject: [PATCH 10/17] Add pert_ prefix for perturbation-only functions. --- mart/attack/composer.py | 8 ++++---- mart/configs/attack/composer/functions/mask.yaml | 4 ---- .../attack/composer/functions/pert_mask.yaml | 4 ++++ .../composer/functions/pert_rect_crop.yaml | 4 ++++ .../attack/composer/functions/pert_rect_pad.yaml | 5 +++++ .../pert_rect_perspective_transform.yaml | 5 +++++ .../attack/composer/functions/rect_crop.yaml | 4 ---- .../attack/composer/functions/rect_pad.yaml | 5 ----- .../functions/rect_perspective_transform.yaml | 5 ----- .../attack/composer/rect_patch_additive.yaml | 16 ++++++++-------- .../attack/composer/rect_patch_overlay.yaml | 12 ++++++------ 11 files changed, 36 insertions(+), 36 deletions(-) delete mode 100644 mart/configs/attack/composer/functions/mask.yaml create mode 100644 mart/configs/attack/composer/functions/pert_mask.yaml create mode 100644 mart/configs/attack/composer/functions/pert_rect_crop.yaml create mode 100644 mart/configs/attack/composer/functions/pert_rect_pad.yaml create mode 100644 mart/configs/attack/composer/functions/pert_rect_perspective_transform.yaml delete mode 100644 mart/configs/attack/composer/functions/rect_crop.yaml delete mode 100644 mart/configs/attack/composer/functions/rect_pad.yaml delete mode 100644 mart/configs/attack/composer/functions/rect_perspective_transform.yaml diff --git a/mart/attack/composer.py b/mart/attack/composer.py index a0e31367..ef8bba85 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -93,7 +93,7 @@ def forward(self, perturbation, input, target): return perturbation, input, target -class Mask(Function): +class PerturbationMask(Function): def __init__(self, *args, key="perturbable_mask", **kwargs): super().__init__(*args, **kwargs) self.key = key @@ -125,7 +125,7 @@ def forward(self, perturbation, input, target): return perturbation, input, target -class RectangleCrop(Function): +class PerturbationRectangleCrop(Function): def __init__(self, *args, coords_key="patch_coords", **kwargs): super().__init__(*args, **kwargs) self.coords_key = coords_key @@ -160,7 +160,7 @@ def forward(self, perturbation, input, target): return rectangle_patch, input, target -class RectanglePad(Function): +class PerturbationRectanglePad(Function): def __init__(self, *args, coords_key="patch_coords", rect_coords_key="rect_coords", **kwargs): super().__init__(*args, **kwargs) self.coords_key = coords_key @@ -193,7 +193,7 @@ def forward(self, perturbation_patch, input, target): return perturbation_padded, input, target -class RectanglePerspectiveTransform(Function): +class PerturbationRectanglePerspectiveTransform(Function): def __init__(self, *args, coords_key="patch_coords", rect_coords_key="rect_coords", **kwargs): super().__init__(*args, **kwargs) self.coords_key = coords_key diff --git a/mart/configs/attack/composer/functions/mask.yaml b/mart/configs/attack/composer/functions/mask.yaml deleted file mode 100644 index 04cefaf8..00000000 --- a/mart/configs/attack/composer/functions/mask.yaml +++ /dev/null @@ -1,4 +0,0 @@ -mask: - _target_: mart.attack.composer.Mask - key: perturbable_mask - order: 0 diff --git a/mart/configs/attack/composer/functions/pert_mask.yaml b/mart/configs/attack/composer/functions/pert_mask.yaml new file mode 100644 index 00000000..c3adb784 --- /dev/null +++ b/mart/configs/attack/composer/functions/pert_mask.yaml @@ -0,0 +1,4 @@ +pert_mask: + _target_: mart.attack.composer.PerturbationMask + key: perturbable_mask + order: 0 diff --git a/mart/configs/attack/composer/functions/pert_rect_crop.yaml b/mart/configs/attack/composer/functions/pert_rect_crop.yaml new file mode 100644 index 00000000..73d4f4b1 --- /dev/null +++ b/mart/configs/attack/composer/functions/pert_rect_crop.yaml @@ -0,0 +1,4 @@ +pert_rect_crop: + _target_: mart.attack.composer.PerturbationRectangleCrop + coords_key: patch_coords + order: 0 diff --git a/mart/configs/attack/composer/functions/pert_rect_pad.yaml b/mart/configs/attack/composer/functions/pert_rect_pad.yaml new file mode 100644 index 00000000..ce45ec19 --- /dev/null +++ b/mart/configs/attack/composer/functions/pert_rect_pad.yaml @@ -0,0 +1,5 @@ +pert_rect_pad: + _target_: mart.attack.composer.PerturbationRectanglePad + coords_key: patch_coords + rect_coords_key: rect_coords + order: 0 diff --git a/mart/configs/attack/composer/functions/pert_rect_perspective_transform.yaml b/mart/configs/attack/composer/functions/pert_rect_perspective_transform.yaml new file mode 100644 index 00000000..0be6f331 --- /dev/null +++ b/mart/configs/attack/composer/functions/pert_rect_perspective_transform.yaml @@ -0,0 +1,5 @@ +pert_rect_perspective_transform: + _target_: mart.attack.composer.PerturbationRectanglePerspectiveTransform + order: 0 + coords_key: patch_coords + rect_coords_key: rect_coords diff --git a/mart/configs/attack/composer/functions/rect_crop.yaml b/mart/configs/attack/composer/functions/rect_crop.yaml deleted file mode 100644 index 011cd6ff..00000000 --- a/mart/configs/attack/composer/functions/rect_crop.yaml +++ /dev/null @@ -1,4 +0,0 @@ -rect_crop: - _target_: mart.attack.composer.RectangleCrop - coords_key: patch_coords - order: 0 diff --git a/mart/configs/attack/composer/functions/rect_pad.yaml b/mart/configs/attack/composer/functions/rect_pad.yaml deleted file mode 100644 index 384a71fd..00000000 --- a/mart/configs/attack/composer/functions/rect_pad.yaml +++ /dev/null @@ -1,5 +0,0 @@ -rect_pad: - _target_: mart.attack.composer.RectanglePad - coords_key: patch_coords - rect_coords_key: rect_coords - order: 0 diff --git a/mart/configs/attack/composer/functions/rect_perspective_transform.yaml b/mart/configs/attack/composer/functions/rect_perspective_transform.yaml deleted file mode 100644 index 87eac864..00000000 --- a/mart/configs/attack/composer/functions/rect_perspective_transform.yaml +++ /dev/null @@ -1,5 +0,0 @@ -rect_perspective_transform: - _target_: mart.attack.composer.RectanglePerspectiveTransform - order: 0 - coords_key: patch_coords - rect_coords_key: rect_coords diff --git a/mart/configs/attack/composer/rect_patch_additive.yaml b/mart/configs/attack/composer/rect_patch_additive.yaml index 5a0247ce..d9171e08 100644 --- a/mart/configs/attack/composer/rect_patch_additive.yaml +++ b/mart/configs/attack/composer/rect_patch_additive.yaml @@ -2,25 +2,25 @@ defaults: - default - functions: [ - rect_crop, + pert_rect_crop, pert_image, - rect_pad, - rect_perspective_transform, - mask, + pert_rect_pad, + pert_rect_perspective_transform, + pert_mask, additive, fake_clamp, ] functions: - rect_crop: + pert_rect_crop: order: 0 pert_image: order: 1 - rect_pad: + pert_rect_pad: order: 2 - rect_perspective_transform: + pert_rect_perspective_transform: order: 3 - mask: + pert_mask: order: 4 additive: order: 5 diff --git a/mart/configs/attack/composer/rect_patch_overlay.yaml b/mart/configs/attack/composer/rect_patch_overlay.yaml index 8e1ab67a..74c5a249 100644 --- a/mart/configs/attack/composer/rect_patch_overlay.yaml +++ b/mart/configs/attack/composer/rect_patch_overlay.yaml @@ -2,22 +2,22 @@ defaults: - default - functions: [ - rect_crop, + pert_rect_crop, pert_image, - rect_pad, - rect_perspective_transform, + pert_rect_pad, + pert_rect_perspective_transform, overlay, fake_clamp, ] functions: - rect_crop: + pert_rect_crop: order: 0 pert_image: order: 1 - rect_pad: + pert_rect_pad: order: 2 - rect_perspective_transform: + pert_rect_perspective_transform: order: 3 overlay: order: 4 From 97c2f3abf963130afaf14a52507ab7364c228a5a Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 12:53:23 -0700 Subject: [PATCH 11/17] Remove pert_image from the patch additive composer. --- .../attack/composer/rect_patch_additive.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mart/configs/attack/composer/rect_patch_additive.yaml b/mart/configs/attack/composer/rect_patch_additive.yaml index d9171e08..be3d8434 100644 --- a/mart/configs/attack/composer/rect_patch_additive.yaml +++ b/mart/configs/attack/composer/rect_patch_additive.yaml @@ -3,7 +3,6 @@ defaults: - functions: [ pert_rect_crop, - pert_image, pert_rect_pad, pert_rect_perspective_transform, pert_mask, @@ -14,15 +13,13 @@ defaults: functions: pert_rect_crop: order: 0 - pert_image: - order: 1 pert_rect_pad: - order: 2 + order: 1 pert_rect_perspective_transform: - order: 3 + order: 2 pert_mask: - order: 4 + order: 3 additive: - order: 5 + order: 4 fake_clamp: - order: 6 + order: 5 From eb8336d78a38622828a42992668a2cee542d9cad Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 12:54:59 -0700 Subject: [PATCH 12/17] Rename to PerturbationImageAdditive. --- mart/attack/composer.py | 2 +- mart/configs/attack/composer/functions/pert_image.yaml | 4 ---- .../attack/composer/functions/pert_image_additive.yaml | 4 ++++ mart/configs/attack/composer/rect_patch_overlay.yaml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) delete mode 100644 mart/configs/attack/composer/functions/pert_image.yaml create mode 100644 mart/configs/attack/composer/functions/pert_image_additive.yaml diff --git a/mart/attack/composer.py b/mart/attack/composer.py index ef8bba85..be418661 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -236,7 +236,7 @@ def forward(self, perturbation, input, target): return perturbation, input, target -class PerturbationImage(Function): +class PerturbationImageAdditive(Function): """Add an image to perturbation if specified.""" def __init__(self, *args, path: str | None = None, scale: int = 1, **kwargs): diff --git a/mart/configs/attack/composer/functions/pert_image.yaml b/mart/configs/attack/composer/functions/pert_image.yaml deleted file mode 100644 index 848a5d5d..00000000 --- a/mart/configs/attack/composer/functions/pert_image.yaml +++ /dev/null @@ -1,4 +0,0 @@ -pert_image: - _target_: mart.attack.composer.PerturbationImage - path: null - order: 0 diff --git a/mart/configs/attack/composer/functions/pert_image_additive.yaml b/mart/configs/attack/composer/functions/pert_image_additive.yaml new file mode 100644 index 00000000..c6e5d9a0 --- /dev/null +++ b/mart/configs/attack/composer/functions/pert_image_additive.yaml @@ -0,0 +1,4 @@ +pert_image_additive: + _target_: mart.attack.composer.PerturbationImageAdditive + path: null + order: 0 diff --git a/mart/configs/attack/composer/rect_patch_overlay.yaml b/mart/configs/attack/composer/rect_patch_overlay.yaml index 74c5a249..21656346 100644 --- a/mart/configs/attack/composer/rect_patch_overlay.yaml +++ b/mart/configs/attack/composer/rect_patch_overlay.yaml @@ -3,7 +3,7 @@ defaults: - functions: [ pert_rect_crop, - pert_image, + pert_image_additive, pert_rect_pad, pert_rect_perspective_transform, overlay, @@ -13,7 +13,7 @@ defaults: functions: pert_rect_crop: order: 0 - pert_image: + pert_image_additive: order: 1 pert_rect_pad: order: 2 From 1994aec732ce4b741bcce98908094a9e5bbb5f16 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 12:56:19 -0700 Subject: [PATCH 13/17] Adjust order in code. --- mart/attack/composer.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/mart/attack/composer.py b/mart/attack/composer.py index be418661..9ab99ce9 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -125,6 +125,26 @@ def forward(self, perturbation, input, target): return perturbation, input, target +class FakeClamp(Function): + """A Clamp operation that preserves gradients.""" + + def __init__(self, *args, min_val, max_val, **kwargs): + super().__init__(*args, **kwargs) + self.min_val = min_val + self.max_val = max_val + + @staticmethod + def fake_clamp(x, *, min_val, max_val): + with torch.no_grad(): + x_clamped = x.clamp(min_val, max_val) + diff = x_clamped - x + return x + diff + + def forward(self, perturbation, input, target): + input = self.fake_clamp(input, min_val=self.min_val, max_val=self.max_val) + return perturbation, input, target + + class PerturbationRectangleCrop(Function): def __init__(self, *args, coords_key="patch_coords", **kwargs): super().__init__(*args, **kwargs) @@ -216,26 +236,6 @@ def forward(self, perturbation_rect, input, target): return perturbation_coords, input, target -class FakeClamp(Function): - """A Clamp operation that preserves gradients.""" - - def __init__(self, *args, min_val, max_val, **kwargs): - super().__init__(*args, **kwargs) - self.min_val = min_val - self.max_val = max_val - - @staticmethod - def fake_clamp(x, *, min_val, max_val): - with torch.no_grad(): - x_clamped = x.clamp(min_val, max_val) - diff = x_clamped - x - return x + diff - - def forward(self, perturbation, input, target): - input = self.fake_clamp(input, min_val=self.min_val, max_val=self.max_val) - return perturbation, input, target - - class PerturbationImageAdditive(Function): """Add an image to perturbation if specified.""" From f7937c0b27b0259e3e9a59e52ab00819bd77387d Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 13:05:13 -0700 Subject: [PATCH 14/17] Fix tests. --- tests/test_composer.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/test_composer.py b/tests/test_composer.py index 93db2abd..96c8614a 100644 --- a/tests/test_composer.py +++ b/tests/test_composer.py @@ -9,11 +9,11 @@ from mart.attack.composer import ( Additive, Composer, - Mask, Overlay, - RectangleCrop, - RectanglePad, - RectanglePerspectiveTransform, + PerturbationMask, + PerturbationRectangleCrop, + PerturbationRectanglePad, + PerturbationRectanglePerspectiveTransform, ) @@ -35,22 +35,24 @@ def test_overlay_composer_forward(input_data, target_data, perturbation): torch.testing.assert_close(output, expected_output, equal_nan=True) -def test_mask_additive_composer_forward(): +def test_pert_mask_additive_composer_forward(): input = torch.zeros((2, 2)) perturbation = torch.ones((2, 2)) target = {"perturbable_mask": torch.eye(2)} expected_output = torch.eye(2) - composer = Composer(functions={"mask": Mask(order=0), "additive": Additive(order=1)}) + composer = Composer( + functions={"pert_mask": PerturbationMask(order=0), "additive": Additive(order=1)} + ) output = composer(perturbation, input=input, target=target) torch.testing.assert_close(output, expected_output, equal_nan=True) -def test_rect_crop(): +def test_pert_rect_crop(): key = "patch_coords" input = torch.zeros((3, 10, 10)) perturbation = torch.ones_like(input) - fn = RectangleCrop(coords_key=key) + fn = PerturbationRectangleCrop(coords_key=key) # FIXME: four corner points (width, height) of a patch in the order of top-left, top-right, bottom-right, bottom-left. # A simple square patch. @@ -72,7 +74,7 @@ def test_rect_crop(): assert rect_patch.shape == (3, 8, 6) -def test_rect_pad(): +def test_pert_rect_pad(): coords_key = "patch_coords" rect_coords_key = "rect_coords" @@ -82,7 +84,7 @@ def test_rect_pad(): input = torch.zeros((3, 10, 10)) target = {coords_key: patch_coords} - fn = RectanglePad(coords_key=coords_key, rect_coords_key=rect_coords_key) + fn = PerturbationRectanglePad(coords_key=coords_key, rect_coords_key=rect_coords_key) pert_padded, _input, _target = fn(rect_patch, input, target) pert_padded_expected = torch.zeros_like(input) @@ -94,7 +96,7 @@ def test_rect_pad(): assert _target[rect_coords_key] == rect_coords_expected -def test_rect_perspective_transform(): +def test_pert_rect_perspective_transform(): coords_key = "patch_coords" rect_coords_key = "rect_coords" @@ -108,7 +110,9 @@ def test_rect_perspective_transform(): pert_padded = torch.zeros_like(input) pert_padded[:, :5, :5] = 1 - fn = RectanglePerspectiveTransform(coords_key=coords_key, rect_coords_key=rect_coords_key) + fn = PerturbationRectanglePerspectiveTransform( + coords_key=coords_key, rect_coords_key=rect_coords_key + ) pert_coords, _input, _target = fn(pert_padded, input, target) pert_coords_expected = torch.zeros_like(input) pert_coords_expected[:, 5:, 5:] = 1 From 884fb481b8586b358e65c84bd194f16c6ea5b0b6 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 13:07:22 -0700 Subject: [PATCH 15/17] Rename to InputFakeClamp. --- mart/attack/composer.py | 3 ++- mart/configs/attack/composer/functions/fake_clamp.yaml | 5 ----- mart/configs/attack/composer/functions/input_fake_clamp.yaml | 5 +++++ 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 mart/configs/attack/composer/functions/fake_clamp.yaml create mode 100644 mart/configs/attack/composer/functions/input_fake_clamp.yaml diff --git a/mart/attack/composer.py b/mart/attack/composer.py index 9ab99ce9..8f41e11b 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -104,6 +104,7 @@ def forward(self, perturbation, input, target): return perturbation, input, target +# TODO: We may decompose Overlay into: perturbation-mask, input-re-mask, additive. class Overlay(Function): """We assume an adversary overlays a patch to the input.""" @@ -125,7 +126,7 @@ def forward(self, perturbation, input, target): return perturbation, input, target -class FakeClamp(Function): +class InputFakeClamp(Function): """A Clamp operation that preserves gradients.""" def __init__(self, *args, min_val, max_val, **kwargs): diff --git a/mart/configs/attack/composer/functions/fake_clamp.yaml b/mart/configs/attack/composer/functions/fake_clamp.yaml deleted file mode 100644 index 50ef4fbe..00000000 --- a/mart/configs/attack/composer/functions/fake_clamp.yaml +++ /dev/null @@ -1,5 +0,0 @@ -fake_clamp: - _target_: mart.attack.composer.FakeClamp - order: 0 - min_val: 0 - max_val: 255 diff --git a/mart/configs/attack/composer/functions/input_fake_clamp.yaml b/mart/configs/attack/composer/functions/input_fake_clamp.yaml new file mode 100644 index 00000000..764f08d4 --- /dev/null +++ b/mart/configs/attack/composer/functions/input_fake_clamp.yaml @@ -0,0 +1,5 @@ +input_fake_clamp: + _target_: mart.attack.composer.InputFakeClamp + order: 0 + min_val: 0 + max_val: 255 From 09753720a16ff8f8bbea0ce9eecdab9a86275cb8 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 13:08:08 -0700 Subject: [PATCH 16/17] Rename input_fake_clamp. --- mart/configs/attack/composer/rect_patch_additive.yaml | 4 ++-- mart/configs/attack/composer/rect_patch_overlay.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mart/configs/attack/composer/rect_patch_additive.yaml b/mart/configs/attack/composer/rect_patch_additive.yaml index be3d8434..ba5a2218 100644 --- a/mart/configs/attack/composer/rect_patch_additive.yaml +++ b/mart/configs/attack/composer/rect_patch_additive.yaml @@ -7,7 +7,7 @@ defaults: pert_rect_perspective_transform, pert_mask, additive, - fake_clamp, + input_fake_clamp, ] functions: @@ -21,5 +21,5 @@ functions: order: 3 additive: order: 4 - fake_clamp: + input_fake_clamp: order: 5 diff --git a/mart/configs/attack/composer/rect_patch_overlay.yaml b/mart/configs/attack/composer/rect_patch_overlay.yaml index 21656346..c773d747 100644 --- a/mart/configs/attack/composer/rect_patch_overlay.yaml +++ b/mart/configs/attack/composer/rect_patch_overlay.yaml @@ -7,7 +7,7 @@ defaults: pert_rect_pad, pert_rect_perspective_transform, overlay, - fake_clamp, + input_fake_clamp, ] functions: @@ -21,5 +21,5 @@ functions: order: 3 overlay: order: 4 - fake_clamp: + input_fake_clamp: order: 5 From 8db66dc27ed9c0fd2163a15ad146a570415444d8 Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 28 Sep 2023 15:52:00 -0700 Subject: [PATCH 17/17] Adjust order. --- mart/attack/composer.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mart/attack/composer.py b/mart/attack/composer.py index e1df942e..7747f958 100644 --- a/mart/attack/composer.py +++ b/mart/attack/composer.py @@ -110,17 +110,6 @@ def forward(self, perturbation, input, target): return perturbation, input, target -class PerturbationMask(Function): - def __init__(self, *args, key="perturbable_mask", **kwargs): - super().__init__(*args, **kwargs) - self.key = key - - def forward(self, perturbation, input, target): - mask = target[self.key] - perturbation = perturbation * mask - return perturbation, input, target - - # TODO: We may decompose Overlay into: perturbation-mask, input-re-mask, additive. class Overlay(Function): """We assume an adversary overlays a patch to the input.""" @@ -163,6 +152,17 @@ def forward(self, perturbation, input, target): return perturbation, input, target +class PerturbationMask(Function): + def __init__(self, *args, key="perturbable_mask", **kwargs): + super().__init__(*args, **kwargs) + self.key = key + + def forward(self, perturbation, input, target): + mask = target[self.key] + perturbation = perturbation * mask + return perturbation, input, target + + class PerturbationRectangleCrop(Function): def __init__(self, *args, coords_key="patch_coords", **kwargs): super().__init__(*args, **kwargs)