From 37d61e11bb11ece36f655b38d9fea01ae27f2a3d Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Tue, 18 Apr 2023 15:14:24 -0700 Subject: [PATCH 01/12] Add YOLOX --- mart/configs/datamodule/coco_yolox.yaml | 36 +++++++++ mart/configs/experiment/COCO_YOLOX.yaml | 34 ++++++++ .../experiment/COCO_YOLOX_ShapeShifter.yaml | 78 +++++++++++++++++++ mart/configs/model/yolox.yaml | 61 +++++++++++++++ mart/datamodules/coco.py | 7 ++ mart/models/yolox.py | 70 +++++++++++++++++ pyproject.toml | 1 + 7 files changed, 287 insertions(+) create mode 100644 mart/configs/datamodule/coco_yolox.yaml create mode 100644 mart/configs/experiment/COCO_YOLOX.yaml create mode 100644 mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml create mode 100644 mart/configs/model/yolox.yaml create mode 100644 mart/models/yolox.py diff --git a/mart/configs/datamodule/coco_yolox.yaml b/mart/configs/datamodule/coco_yolox.yaml new file mode 100644 index 00000000..69f6f544 --- /dev/null +++ b/mart/configs/datamodule/coco_yolox.yaml @@ -0,0 +1,36 @@ +defaults: + - default.yaml + +num_workers: 2 + +train_dataset: + _target_: yolox.data.datasets.coco.COCODataset + data_dir: ${paths.data_dir}/coco + name: train2017 + json_file: instances_train2017.json + img_size: ??? + preproc: + _target_: yolox.data.TrainTransform + max_labels: 50 + flip_prob: 0.5 + hsv_prob: 1.0 + +val_dataset: + _target_: yolox.data.datasets.coco.COCODataset + data_dir: ${paths.data_dir}/coco + name: val2017 + json_file: instances_val2017.json + img_size: ${..train_dataset.img_size} + preproc: + # Use TrainTransform instead of ValTransform since it supplies targets + # we can use to measure. + _target_: yolox.data.TrainTransform + max_labels: 100 + flip_prob: 0. + hsv_prob: 0. + +test_dataset: ${.val_dataset} + +collate_fn: + _target_: hydra.utils.get_method + path: mart.datamodules.coco.collate_yolox_fn diff --git a/mart/configs/experiment/COCO_YOLOX.yaml b/mart/configs/experiment/COCO_YOLOX.yaml new file mode 100644 index 00000000..d23f2366 --- /dev/null +++ b/mart/configs/experiment/COCO_YOLOX.yaml @@ -0,0 +1,34 @@ +# @package _global_ + +defaults: + - override /datamodule: coco_yolox + - override /model: yolox + - override /metric: average_precision + - override /optimization: super_convergence + +task_name: "COCO_YOLOX" +tags: ["evaluation"] + +optimized_metric: "test_metrics/map" + +trainer: + # 117,266 training images, 6 epochs, batch_size=16, 43,974.75 + max_steps: 43975 + # FIXME: "nms_kernel" not implemented for 'BFloat16', torch.ops.torchvision.nms(). + precision: 32 + +datamodule: + num_workers: 32 + ims_per_batch: 16 + + train_dataset: + img_size: [416, 416] + +model: + # YOLOX model does not produce preds/targets in training sequence + training_metrics: null + + optimizer: + lr: 0.001 + momentum: 0.9 + weight_decay: 0.0005 diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml new file mode 100644 index 00000000..a0cf6e52 --- /dev/null +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -0,0 +1,78 @@ +# @package _global_ + +defaults: + - COCO_YOLOX + - /attack/perturber@model.modules.perturber: default + - /attack/perturber/initializer@model.modules.perturber.initializer: uniform + - /attack/perturber/composer@model.modules.perturber.composer: color_jitter_random_affine_overlay + - /attack/perturber/projector@model.modules.perturber.projector: range + - /attack/optimizer@model.optimizer: sgd + - /attack/gradient_modifier@model.gradient_modifier: lp_normalizer + - override /callbacks: [perturbation_visualizer, lr_monitor] + +task_name: "COCO_YOLOv3_ShapeShifter" +tags: ["adv"] + +trainer: + # 118287 training images, 1 epoch, batch_size=16: FLOOR(118287/16) = 7392 + max_steps: 73920 + # mAP can be slow to compute so limit number of images + limit_val_batches: 100 + limit_test_batches: 100 + +callbacks: + perturbation_visualizer: + frequency: 500 + +datamodule: + num_workers: 32 + ims_per_batch: 16 + +model: + modules: + perturber: + size: [3, 416, 234] + + initializer: + min: 0.49 + max: 0.51 + + projector: + min: 0.0 + max: 1.0 + + composer: + degrees: [-5, 5] + translate: null + scale: [0.3, 0.5] + shear: [-3, 3, -3, 3] + brightness: [0.5, 1.5] + contrast: [0.5, 1.5] + saturation: [0.5, 1.5] + hue: [-0.05, 0.05] + pixel_scale: 1.0 + clamp: [0, 1] + + freeze: "logits" + + optimizer: + lr: 0.01 + momentum: 0.9 + + gradient_modifier: + p: inf + + training_sequence: + seq005: perturber + seq010: + logits: ["perturber"] + + validation_sequence: + seq005: perturber + seq010: + logits: ["perturber"] + + test_sequence: + seq005: perturber + seq010: + logits: ["perturber"] diff --git a/mart/configs/model/yolox.yaml b/mart/configs/model/yolox.yaml new file mode 100644 index 00000000..081be225 --- /dev/null +++ b/mart/configs/model/yolox.yaml @@ -0,0 +1,61 @@ +defaults: + - modular + +modules: + losses_or_predictions: + _target_: yolox.models.build.create_yolox_model + name: "yolov3" + pretrained: True + num_classes: 80 + device: "cpu" # PL handles this + + loss: + _target_: mart.nn.Sum + + detections: + _target_: mart.models.yolox.Detections + conf_thre: 0.1 + nms_thre: 0.4 + + output: + _target_: mart.nn.ReturnKwargs + +training_metrics: null + +training_sequence: + seq010: + losses_or_predictions: + x: input + targets: target + + seq020: + loss: + - losses_or_predictions.total_loss + + seq030: + output: + loss: loss + total_loss: losses_or_predictions.total_loss + iou_loss: losses_or_predictions.iou_loss + l1_loss: losses_or_predictions.l1_loss + conf_loss: losses_or_predictions.conf_loss + cls_loss: losses_or_predictions.cls_loss + num_fg: losses_or_predictions.num_fg + +validation_sequence: + seq010: + losses_or_predictions: + x: "input" + targets: target + + seq020: + detections: + predictions: losses_or_predictions + targets: target + + seq030: + output: + preds: detections.preds + target: detections.target + +test_sequence: ${.validation_sequence} diff --git a/mart/datamodules/coco.py b/mart/datamodules/coco.py index 42ddcebb..30ab95da 100644 --- a/mart/datamodules/coco.py +++ b/mart/datamodules/coco.py @@ -8,6 +8,7 @@ from typing import Any, Callable, List, Optional import numpy as np +from torch.utils.data import default_collate from torchvision.datasets.coco import CocoDetection as CocoDetection_ from torchvision.datasets.folder import default_loader @@ -89,3 +90,9 @@ def __getitem__(self, index: int): # Source: https://github.com/pytorch/vision/blob/dc07ac2add8285e16a716564867d0b4b953f6735/references/detection/utils.py#L203 def collate_fn(batch): return tuple(zip(*batch)) + + +def collate_yolox_fn(batch): + batch = default_collate(batch) + image, target, *_ = batch + return image, target diff --git a/mart/models/yolox.py b/mart/models/yolox.py new file mode 100644 index 00000000..b73b3f71 --- /dev/null +++ b/mart/models/yolox.py @@ -0,0 +1,70 @@ +# +# Copyright (C) 2023 Intel Corporation +# +# SPDX-License-Identifier: BSD-3-Clause +# + +import torch +from yolox.utils import postprocess + + +class Detections(torch.nn.Module): + def __init__(self, num_classes=80, conf_thre=0.7, nms_thre=0.45, class_agnostic=False): + super().__init__() + + self.num_classes = num_classes + self.conf_thre = conf_thre + self.nms_thre = nms_thre + self.class_agnostic = class_agnostic + + @staticmethod + def cxcywh2xyxy(bboxes): + bboxes[:, 0] = bboxes[:, 0] - bboxes[:, 2] * 0.5 + bboxes[:, 1] = bboxes[:, 1] - bboxes[:, 3] * 0.5 + bboxes[:, 2] = bboxes[:, 0] + bboxes[:, 2] + bboxes[:, 3] = bboxes[:, 1] + bboxes[:, 3] + return bboxes + + @staticmethod + def tensor_to_dict(detection): + if detection is None: + # Handle images with no detections + boxes = torch.empty((0, 4), device="cuda") # HACK + labels = torch.empty((0,), device="cuda") # HACK + scores = torch.empty((0,), device="cuda") # HACK + + elif detection.shape[1] > 5: + # (x1, y1, x2, y2, obj_conf, class_conf, class_pred) + boxes = detection[:, 0:4] + labels = detection[:, 6].to(int) + scores = detection[:, 4] * detection[:, 5] + + else: # targets have no scores + # [class, xc, yc, w, h] + boxes = detection[:, 1:5] + boxes = Detections.cxcywh2xyxy(boxes) + labels = detection[:, 0].to(int) + scores = torch.ones_like(labels) + + length = (labels > 0).sum() + + boxes = boxes[:length] + labels = labels[:length] + scores = scores[:length] + + return {"boxes": boxes, "labels": labels, "scores": scores} + + def forward(self, predictions, targets): + detections = postprocess( + predictions, + self.num_classes, + conf_thre=self.conf_thre, + nms_thre=self.nms_thre, + class_agnostic=self.class_agnostic, + ) + + # Convert preds and targets to format acceptable to torchmetrics + preds = [Detections.tensor_to_dict(det) for det in detections] + targets = [Detections.tensor_to_dict(tar) for tar in targets] + + return {"preds": preds, "target": targets} diff --git a/pyproject.toml b/pyproject.toml index 6440af7b..94adeda4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ # ----- object detection----- # "pycocotools ~= 2.0.5", + "yolox @ git+https://github.com/Megvii-BaseDetection/YOLOX.git@0.3.0", # -------- Adversary ---------# "robustbench @ git+https://github.com/RobustBench/robustbench.git@9a590683b7daecf963244dea402529f0d728c727", From d2183ab86f8df73712f1d04a56e45c59076c34f2 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Tue, 18 Apr 2023 16:06:49 -0700 Subject: [PATCH 02/12] style --- mart/models/yolox.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mart/models/yolox.py b/mart/models/yolox.py index b73b3f71..99c77962 100644 --- a/mart/models/yolox.py +++ b/mart/models/yolox.py @@ -29,9 +29,9 @@ def cxcywh2xyxy(bboxes): def tensor_to_dict(detection): if detection is None: # Handle images with no detections - boxes = torch.empty((0, 4), device="cuda") # HACK - labels = torch.empty((0,), device="cuda") # HACK - scores = torch.empty((0,), device="cuda") # HACK + boxes = torch.empty((0, 4), device="cuda") # HACK + labels = torch.empty((0,), device="cuda") # HACK + scores = torch.empty((0,), device="cuda") # HACK elif detection.shape[1] > 5: # (x1, y1, x2, y2, obj_conf, class_conf, class_pred) From 1eda56da9140d6625091a22c43714da1376a2566 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Tue, 18 Apr 2023 16:12:40 -0700 Subject: [PATCH 03/12] bugfix --- .../experiment/COCO_YOLOX_ShapeShifter.yaml | 14 +++++++------- mart/configs/model/yolox.yaml | 6 +++--- mart/datamodules/coco.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml index a0cf6e52..d0b4ba3d 100644 --- a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -65,14 +65,14 @@ model: training_sequence: seq005: perturber seq010: - logits: ["perturber"] + losses_or_predictions: + x: perturber + targets: target.target - validation_sequence: - seq005: perturber - seq010: - logits: ["perturber"] - test_sequence: + validation_sequence: seq005: perturber seq010: - logits: ["perturber"] + losses_or_predictions: + x: perturber + targets: target.target diff --git a/mart/configs/model/yolox.yaml b/mart/configs/model/yolox.yaml index 081be225..56dd0e59 100644 --- a/mart/configs/model/yolox.yaml +++ b/mart/configs/model/yolox.yaml @@ -45,13 +45,13 @@ training_sequence: validation_sequence: seq010: losses_or_predictions: - x: "input" - targets: target + x: input + targets: target.target seq020: detections: predictions: losses_or_predictions - targets: target + targets: target.target seq030: output: diff --git a/mart/datamodules/coco.py b/mart/datamodules/coco.py index 30ab95da..96b40bd9 100644 --- a/mart/datamodules/coco.py +++ b/mart/datamodules/coco.py @@ -95,4 +95,4 @@ def collate_fn(batch): def collate_yolox_fn(batch): batch = default_collate(batch) image, target, *_ = batch - return image, target + return image, {"target": target} From 599dc883ff35a6076b764f19fac8c792e3a7fc29 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Wed, 19 Apr 2023 08:14:34 -0700 Subject: [PATCH 04/12] Update warp --- .../experiment/COCO_YOLOX_ShapeShifter.yaml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml index d0b4ba3d..17460e32 100644 --- a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -4,7 +4,7 @@ defaults: - COCO_YOLOX - /attack/perturber@model.modules.perturber: default - /attack/perturber/initializer@model.modules.perturber.initializer: uniform - - /attack/perturber/composer@model.modules.perturber.composer: color_jitter_random_affine_overlay + - /attack/perturber/composer@model.modules.perturber.composer: color_jitter_warp_overlay - /attack/perturber/projector@model.modules.perturber.projector: range - /attack/optimizer@model.optimizer: sgd - /attack/gradient_modifier@model.gradient_modifier: lp_normalizer @@ -42,16 +42,21 @@ model: max: 1.0 composer: - degrees: [-5, 5] - translate: null - scale: [0.3, 0.5] - shear: [-3, 3, -3, 3] + warp: + _target_: torchvision.transforms.Compose + transforms: + - _target_: torchvision.transforms.RandomAffine + degrees: 0 + scale: [0.3, 0.5] + - _target_: torchvision.transforms.RandomPerspective + distortion_scale: 0.2 + p: 0.5 + clamp: [0, 255] brightness: [0.5, 1.5] contrast: [0.5, 1.5] saturation: [0.5, 1.5] hue: [-0.05, 0.05] pixel_scale: 1.0 - clamp: [0, 1] freeze: "logits" From d4743f33dbfc14014916a0c49923f30657f1e300 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Wed, 19 Apr 2023 16:25:37 -0700 Subject: [PATCH 05/12] bugfix --- mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml index 17460e32..8c9a11a1 100644 --- a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -45,6 +45,10 @@ model: warp: _target_: torchvision.transforms.Compose transforms: + - _target_: torchvision.transforms.RandomErasing + p: 0.75 + scale: [0.2, 0.7] + ratio: [0.3, 3.3] - _target_: torchvision.transforms.RandomAffine degrees: 0 scale: [0.3, 0.5] @@ -58,7 +62,7 @@ model: hue: [-0.05, 0.05] pixel_scale: 1.0 - freeze: "logits" + freeze: "losses_or_predictions" optimizer: lr: 0.01 From 866c0e7ff0ba4b0374986bea3711d8e58fb9e15a Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Wed, 19 Apr 2023 17:34:48 -0700 Subject: [PATCH 06/12] bugfix --- mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml index 8c9a11a1..2aef0dda 100644 --- a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -10,7 +10,7 @@ defaults: - /attack/gradient_modifier@model.gradient_modifier: lp_normalizer - override /callbacks: [perturbation_visualizer, lr_monitor] -task_name: "COCO_YOLOv3_ShapeShifter" +task_name: "COCO_YOLOX_ShapeShifter" tags: ["adv"] trainer: From 05e7f4d24902766216eec9bf5f6ebe660821c3d2 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Wed, 19 Apr 2023 17:59:32 -0700 Subject: [PATCH 07/12] bugfix --- mart/configs/attack/optimizer/adam.yaml | 10 +++++++ .../experiment/COCO_YOLOX_ShapeShifter.yaml | 24 +++++++++++------ mart/configs/model/yolox.yaml | 26 ++++++++++++++++++- 3 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 mart/configs/attack/optimizer/adam.yaml diff --git a/mart/configs/attack/optimizer/adam.yaml b/mart/configs/attack/optimizer/adam.yaml new file mode 100644 index 00000000..816eadeb --- /dev/null +++ b/mart/configs/attack/optimizer/adam.yaml @@ -0,0 +1,10 @@ +_target_: mart.optim.OptimizerFactory +optimizer: + _target_: hydra.utils.get_method + path: torch.optim.Adam +lr: ??? +betas: [0.9, 0.999] +weight_decay: 0 +bias_decay: 0 +norm_decay: 0 +maximize: True diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml index 2aef0dda..48135a0b 100644 --- a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -1,21 +1,25 @@ # @package _global_ defaults: - - COCO_YOLOX - /attack/perturber@model.modules.perturber: default - /attack/perturber/initializer@model.modules.perturber.initializer: uniform - /attack/perturber/composer@model.modules.perturber.composer: color_jitter_warp_overlay - /attack/perturber/projector@model.modules.perturber.projector: range - - /attack/optimizer@model.optimizer: sgd + - /attack/optimizer@model.optimizer: adam - /attack/gradient_modifier@model.gradient_modifier: lp_normalizer + - override /datamodule: coco_yolox + - override /model: yolox + - override /metric: average_precision - override /callbacks: [perturbation_visualizer, lr_monitor] task_name: "COCO_YOLOX_ShapeShifter" tags: ["adv"] +optimized_metric: "test_metrics/map" + trainer: - # 118287 training images, 1 epoch, batch_size=16: FLOOR(118287/16) = 7392 - max_steps: 73920 + # 118287 training images, 16 epoch, batch_size=16, 16*FLOOR(118287/16) = 7392 + max_steps: 118272 # mAP can be slow to compute so limit number of images limit_val_batches: 100 limit_test_batches: 100 @@ -28,6 +32,9 @@ datamodule: num_workers: 32 ims_per_batch: 16 + train_dataset: + img_size: [416, 416] + model: modules: perturber: @@ -65,11 +72,12 @@ model: freeze: "losses_or_predictions" optimizer: - lr: 0.01 - momentum: 0.9 + lr: 5e-3 + + gradient_modifier: null + # p: inf - gradient_modifier: - p: inf + training_metrics: null training_sequence: seq005: perturber diff --git a/mart/configs/model/yolox.yaml b/mart/configs/model/yolox.yaml index 56dd0e59..52304018 100644 --- a/mart/configs/model/yolox.yaml +++ b/mart/configs/model/yolox.yaml @@ -58,4 +58,28 @@ validation_sequence: preds: detections.preds target: detections.target -test_sequence: ${.validation_sequence} +test_sequence: + seq010: + losses_or_predictions: + x: input + targets: target.target + + seq020: + detections: + predictions: losses_or_predictions + targets: target.target + + seq030: + output: + preds: detections.preds + target: detections.target + + +training_step_log: + - loss + - total_loss + - iou_loss + - l1_loss + - conf_loss + - cls_loss + - num_fg From 01f4981b2b3c91b5d3be3802e0641b470eafc8f0 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Thu, 20 Apr 2023 09:38:06 -0700 Subject: [PATCH 08/12] bugfix --- mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml index 48135a0b..4c753cc5 100644 --- a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -73,6 +73,7 @@ model: optimizer: lr: 5e-3 + maximize: True gradient_modifier: null # p: inf From 77870975dea62f08986183fe350de82921d9445e Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 21 Apr 2023 09:25:08 -0700 Subject: [PATCH 09/12] Perturber yields targets --- mart/attack/perturber.py | 5 ++- .../experiment/COCO_YOLOX_ShapeShifter.yaml | 43 ++++++++++++------- mart/configs/model/yolox.yaml | 1 - 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/mart/attack/perturber.py b/mart/attack/perturber.py index 20072c64..30e4e3eb 100644 --- a/mart/attack/perturber.py +++ b/mart/attack/perturber.py @@ -116,4 +116,7 @@ def forward(self, **batch): ) ) - return {"input_adv": input_adv, "total_variation": total_variation} + targets = batch["target"]["target"] + targets = torch.zeros_like(targets) + + return {"input_adv": input_adv, "total_variation": total_variation, "targets": targets} diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml index 4c753cc5..c41b22ad 100644 --- a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -5,8 +5,8 @@ defaults: - /attack/perturber/initializer@model.modules.perturber.initializer: uniform - /attack/perturber/composer@model.modules.perturber.composer: color_jitter_warp_overlay - /attack/perturber/projector@model.modules.perturber.projector: range - - /attack/optimizer@model.optimizer: adam - /attack/gradient_modifier@model.gradient_modifier: lp_normalizer + - override /optimization: super_convergence - override /datamodule: coco_yolox - override /model: yolox - override /metric: average_precision @@ -18,11 +18,12 @@ tags: ["adv"] optimized_metric: "test_metrics/map" trainer: - # 118287 training images, 16 epoch, batch_size=16, 16*FLOOR(118287/16) = 7392 - max_steps: 118272 + # 118287 training images, batch_size=16, FLOOR(118287/16) = 7392 + max_steps: 73920 # 10 epochs # mAP can be slow to compute so limit number of images limit_val_batches: 100 limit_test_batches: 100 + precision: 32 callbacks: perturbation_visualizer: @@ -57,26 +58,26 @@ model: scale: [0.2, 0.7] ratio: [0.3, 3.3] - _target_: torchvision.transforms.RandomAffine - degrees: 0 + degrees: [-5, 5] scale: [0.3, 0.5] - - _target_: torchvision.transforms.RandomPerspective - distortion_scale: 0.2 - p: 0.5 - clamp: [0, 255] + shear: [-3, 3, -3, 3] + interpolation: 2 # BILINEAR + clamp: [0.0, 1.0] brightness: [0.5, 1.5] contrast: [0.5, 1.5] saturation: [0.5, 1.5] hue: [-0.05, 0.05] pixel_scale: 1.0 + loss: + weights: [-1, 1] # maximize total_loss and minimize total variation freeze: "losses_or_predictions" optimizer: - lr: 5e-3 - maximize: True + lr: 0.01 + momentum: 0.9 gradient_modifier: null - # p: inf training_metrics: null @@ -84,13 +85,23 @@ model: seq005: perturber seq010: losses_or_predictions: - x: perturber - targets: target.target - + x: perturber.input_adv + targets: perturber.targets + seq020: + loss: + - losses_or_predictions.total_loss + - perturber.total_variation validation_sequence: seq005: perturber seq010: losses_or_predictions: - x: perturber - targets: target.target + x: perturber.input_adv + targets: perturber.targets + + test_sequence: + seq005: perturber + seq010: + losses_or_predictions: + x: perturber.input_adv + targets: perturber.targets diff --git a/mart/configs/model/yolox.yaml b/mart/configs/model/yolox.yaml index 52304018..496557fd 100644 --- a/mart/configs/model/yolox.yaml +++ b/mart/configs/model/yolox.yaml @@ -74,7 +74,6 @@ test_sequence: preds: detections.preds target: detections.target - training_step_log: - loss - total_loss From 106ee47b99a97992dee43348cf3c2ef594147eef Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 21 Apr 2023 10:00:22 -0700 Subject: [PATCH 10/12] Log total variation --- .../experiment/COCO_YOLOX_ShapeShifter.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml index c41b22ad..a2d85d2b 100644 --- a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -92,6 +92,20 @@ model: - losses_or_predictions.total_loss - perturber.total_variation + seq030: + output: + total_variation: perturber.total_variation + + training_step_log: + - loss + - total_loss + - iou_loss + - l1_loss + - conf_loss + - cls_loss + - num_fg + - total_variation + validation_sequence: seq005: perturber seq010: From 8bfcc203999baccf3dbd50e67d9284b4f8a6cae2 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Sun, 23 Apr 2023 17:25:05 -0700 Subject: [PATCH 11/12] Add pixel scale to PerturbationVisualizer --- mart/callbacks/visualizer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mart/callbacks/visualizer.py b/mart/callbacks/visualizer.py index 74490dc5..c59cf17d 100644 --- a/mart/callbacks/visualizer.py +++ b/mart/callbacks/visualizer.py @@ -42,12 +42,13 @@ def on_train_end(self, trainer, model): class PerturbationVisualizer(Callback): - def __init__(self, frequency: int = 100): + def __init__(self, frequency: int = 100, pixel_scale: float = 1.0): self.frequency = frequency + self.pixel_scale = pixel_scale def log_perturbation(self, trainer, pl_module): # FIXME: Generalize this by using DotDict? - perturbation = pl_module.model.perturber.perturbation + perturbation = pl_module.model.perturber.perturbation / self.pixel_scale # Add image to each logger for logger in trainer.loggers: From 72ae9fdddff67c8816b22d539eb3f4d26fff9162 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Sun, 23 Apr 2023 17:25:22 -0700 Subject: [PATCH 12/12] Update YOLOX ShapeShifter parameters --- .../experiment/COCO_YOLOX_ShapeShifter.yaml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml index a2d85d2b..39d6bbcf 100644 --- a/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOX_ShapeShifter.yaml @@ -10,7 +10,7 @@ defaults: - override /datamodule: coco_yolox - override /model: yolox - override /metric: average_precision - - override /callbacks: [perturbation_visualizer, lr_monitor] + - override /callbacks: [perturbation_visualizer, lr_monitor, gradient_monitor] task_name: "COCO_YOLOX_ShapeShifter" tags: ["adv"] @@ -28,6 +28,7 @@ trainer: callbacks: perturbation_visualizer: frequency: 500 + pixel_scale: 255 datamodule: num_workers: 32 @@ -42,12 +43,8 @@ model: size: [3, 416, 234] initializer: - min: 0.49 - max: 0.51 - - projector: - min: 0.0 - max: 1.0 + min: 127 + max: 129 composer: warp: @@ -62,19 +59,19 @@ model: scale: [0.3, 0.5] shear: [-3, 3, -3, 3] interpolation: 2 # BILINEAR - clamp: [0.0, 1.0] + clamp: [0, 255] brightness: [0.5, 1.5] contrast: [0.5, 1.5] saturation: [0.5, 1.5] hue: [-0.05, 0.05] - pixel_scale: 1.0 + pixel_scale: 255 loss: - weights: [-1, 1] # maximize total_loss and minimize total variation + weights: [1, 0.00001] # minimize total_loss and total variation freeze: "losses_or_predictions" optimizer: - lr: 0.01 + lr: 25.5 momentum: 0.9 gradient_modifier: null