From b117727417dc07060ff9a3f6acaeeca6f9ad5648 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 9 Jun 2023 09:39:47 -0700 Subject: [PATCH 01/23] WIP --- .../experiment/COCO_YOLOv4_ShapeShifter.yaml | 145 ++++++++++++++ mart/configs/model/yolov4.yaml | 185 ++++++++++++++++++ 2 files changed, 330 insertions(+) create mode 100644 mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml create mode 100644 mart/configs/model/yolov4.yaml diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml new file mode 100644 index 00000000..c206914f --- /dev/null +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -0,0 +1,145 @@ +# @package _global_ + +defaults: + - /attack/perturber@model.modules.perturbation: default + - /attack/perturber/initializer@model.modules.perturbation.initializer: uniform + - /attack/perturber/projector@model.modules.perturbation.projector: range + - /attack/composer@model.modules.input_adv: warp_composite + - /attack/gradient_modifier@model.gradient_modifier: lp_normalizer + - override /optimization: super_convergence + - override /datamodule: coco_yolov3 + - override /model: yolov4 + - override /metric: average_precision + - override /callbacks: + [ + model_checkpoint, + lr_monitor, + perturbation_visualizer, + gradient_monitor, + freeze, + ] + +task_name: "COCO_YOLOv4_ShapeShifter" +tags: ["adv"] + +optimized_metric: "test_metrics/map" + +trainer: + # 64115 training images, batch_size=16, FLOOR(64115/16) = 4007 + max_steps: 40070 # 10 epochs + # mAP can be slow to compute so limit number of images + limit_val_batches: 100 + limit_test_batches: 100 + precision: 32 + +callbacks: + model_checkpoint: + monitor: "validation_metrics/map" + mode: "min" + + freeze: + module: "yolov4" + + perturbation_visualizer: + perturbation: "model.perturbation.perturbation" + frequency: 500 + +datamodule: + num_workers: 32 + ims_per_batch: 16 + + train_dataset: + annFile: ${paths.data_dir}/coco/annotations/person_instances_train2017.json + val_dataset: + annFile: ${paths.data_dir}/coco/annotations/person_instances_val2017.json + test_dataset: + annFile: ${paths.data_dir}/coco/annotations/person_instances_val2017.json + +model: + modules: + perturbation: + size: [3, 416, 234] + + initializer: + min: 0.49 + max: 0.51 + + projector: + min: 0.0 + max: 1.0 + + input_adv: + warp: + _target_: torchvision.transforms.Compose + transforms: + - _target_: mart.transforms.ColorJitter + brightness: [0.5, 1.5] + contrast: [0.5, 1.5] + saturation: [0.5, 1.0] + hue: [-0.05, 0.05] + - _target_: torchvision.transforms.RandomAffine + degrees: [-5, 5] + translate: [0.1, 0.25] + scale: [0.4, 0.6] + shear: [-3, 3, -3, 3] + interpolation: 2 # BILINEAR + clamp: [0, 1] + + loss: + weights: [1, 1] + + optimizer: + lr: 0.01 + momentum: 0.9 + + gradient_modifier: null + + training_sequence: + seq005: perturbation + seq006: input_adv + seq010: + yolov4: + x: "input_adv" + seq030: + loss: + - losses.hide_target_objects_loss + - losses.correct_target_class_loss + + training_step_log: + - loss + - total_loss + - coord_loss + - obj_loss + - noobj_loss + - class_loss + - hide_objects_loss + - target_class_loss + - hide_target_objects_loss + - correct_target_class_loss + - target_count + - score_count + - target_score_count + + training_metrics: null + + validation_sequence: + seq005: perturbation + seq006: input_adv + seq010: + yolov4: + x: "input_adv" + seq030: + loss: + - losses.hide_target_objects_loss + - losses.correct_target_class_loss + + test_sequence: + seq005: perturbation + seq006: input_adv + seq010: + yolov4: + x: "input_adv" + seq030: + loss: + - losses.hide_target_objects_loss + - losses.correct_target_class_loss diff --git a/mart/configs/model/yolov4.yaml b/mart/configs/model/yolov4.yaml new file mode 100644 index 00000000..364c5f6b --- /dev/null +++ b/mart/configs/model/yolov4.yaml @@ -0,0 +1,185 @@ +defaults: + - modular + +modules: + yolov4: + # XXX: This is so sketchy... + _target_: torch.hub.load + repo_or_dir: "AlexeyAB/Yet-Another-YOLOv4-Pytorch" + model: "yolov4" + pretrained: True + trust_repo: False + + losses: + _target_: mart.models.yolov3.Loss + image_size: 416 # FIXME: use ${training_data.transform.image_size}? + average: True + + loss: + _target_: mart.nn.Sum + + detections: + _target_: mart.models.yolov3.Detections + nms: true + conf_thres: 0.1 + nms_thres: 0.4 + + output: + _target_: mart.nn.ReturnKwargs + +# training sequence does not produce preds/targets +training_metrics: null + +training_sequence: + seq010: + yolov4: + x: "input" + + seq020: + losses: + logits: yolov4.logits + target: target + + seq030: + loss: + - losses.total_loss + + seq040: + detections: + preds: yolov4.preds + target: target + + seq050: + output: + loss: loss + total_loss: losses.total_loss + coord_loss: losses.coord_loss + obj_loss: losses.obj_loss + noobj_loss: losses.noobj_loss + class_loss: losses.class_loss + hide_objects_loss: losses.hide_objects_loss + target_class_loss: losses.target_class_loss + hide_target_objects_loss: losses.hide_target_objects_loss + correct_target_class_loss: losses.correct_target_class_loss + target_count: losses.target_count + score_count: losses.score_count + target_score_count: losses.target_score_count + +validation_sequence: + seq010: + yolov4: + x: "input" + + seq020: + losses: + logits: yolov4.logits + target: target + + seq030: + loss: + - losses.total_loss + + seq040: + detections: + preds: yolov4.preds + target: target + + seq050: + output: + preds: detections.preds + target: detections.targets + loss: loss + total_loss: losses.total_loss + coord_loss: losses.coord_loss + obj_loss: losses.obj_loss + noobj_loss: losses.noobj_loss + class_loss: losses.class_loss + hide_objects_loss: losses.hide_objects_loss + target_class_loss: losses.target_class_loss + hide_target_objects_loss: losses.hide_target_objects_loss + correct_target_class_loss: losses.correct_target_class_loss + target_count: losses.target_count + score_count: losses.score_count + target_score_count: losses.target_score_count + +test_sequence: + seq010: + yolov4: + x: "input" + + seq020: + losses: + logits: yolov4.logits + target: target + + seq030: + loss: + - losses.total_loss + + seq040: + detections: + preds: yolov4.preds + target: target + + seq050: + output: + preds: detections.preds + target: detections.targets + loss: loss + total_loss: losses.total_loss + coord_loss: losses.coord_loss + obj_loss: losses.obj_loss + noobj_loss: losses.noobj_loss + class_loss: losses.class_loss + hide_objects_loss: losses.hide_objects_loss + target_class_loss: losses.target_class_loss + hide_target_objects_loss: losses.hide_target_objects_loss + correct_target_class_loss: losses.correct_target_class_loss + target_count: losses.target_count + score_count: losses.score_count + target_score_count: losses.target_score_count + +training_step_log: + - loss + - total_loss + - coord_loss + - obj_loss + - noobj_loss + - class_loss + - hide_objects_loss + - target_class_loss + - hide_target_objects_loss + - correct_target_class_loss + - target_count + - score_count + - target_score_count + +validation_step_log: + - loss + - total_loss + - coord_loss + - obj_loss + - noobj_loss + - class_loss + - hide_objects_loss + - target_class_loss + - hide_target_objects_loss + - correct_target_class_loss + - target_count + - score_count + - target_score_count + +test_step_log: + - loss + - total_loss + - coord_loss + - obj_loss + - noobj_loss + - class_loss + - hide_objects_loss + - target_class_loss + - hide_target_objects_loss + - correct_target_class_loss + - target_count + - score_count + - target_score_count From 8fbc0be1dca237a3db4d7d718e549bd351a29ec0 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Thu, 15 Jun 2023 17:16:21 -0700 Subject: [PATCH 02/23] Fix callbacks --- .../configs/experiment/COCO_YOLOv4_ShapeShifter.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml index c206914f..fdeeecf1 100644 --- a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -16,7 +16,8 @@ defaults: lr_monitor, perturbation_visualizer, gradient_monitor, - freeze, + attack_in_eval_mode, + no_grad_mode, ] task_name: "COCO_YOLOv4_ShapeShifter" @@ -37,8 +38,13 @@ callbacks: monitor: "validation_metrics/map" mode: "min" - freeze: - module: "yolov4" + attack_in_eval_mode: + module_classes: + - _target_: hydra.utils.get_class + path: torch.nn.BatchNorm2d + + no_grad_mode: + module_names: "model.yolov4" perturbation_visualizer: perturbation: "model.perturbation.perturbation" From c7e011ad8ee6e9b07e9a844ae66754eedf200797 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Thu, 15 Jun 2023 17:16:38 -0700 Subject: [PATCH 03/23] Use _return_as_dict_ --- mart/configs/model/yolov4.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mart/configs/model/yolov4.yaml b/mart/configs/model/yolov4.yaml index 364c5f6b..7122b2d3 100644 --- a/mart/configs/model/yolov4.yaml +++ b/mart/configs/model/yolov4.yaml @@ -33,6 +33,7 @@ training_metrics: null training_sequence: seq010: yolov4: + _return_as_dict_: ["logits", "loss"] x: "input" seq020: @@ -68,6 +69,7 @@ training_sequence: validation_sequence: seq010: yolov4: + _return_as_dict_: ["logits", "loss"] x: "input" seq020: @@ -105,6 +107,7 @@ validation_sequence: test_sequence: seq010: yolov4: + _return_as_dict_: ["logits", "loss"] x: "input" seq020: From 1b2d76ec3937c37ab8ddcdb098c6c91a0d6c05f7 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Thu, 15 Jun 2023 17:16:47 -0700 Subject: [PATCH 04/23] Fix losses --- .../experiment/COCO_YOLOv4_ShapeShifter.yaml | 15 +++++++++------ mart/configs/model/yolov4.yaml | 9 ++++++--- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml index fdeeecf1..b37e34f1 100644 --- a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -108,8 +108,9 @@ model: x: "input_adv" seq030: loss: - - losses.hide_target_objects_loss - - losses.correct_target_class_loss + _call_with_args_: + - losses.hide_target_objects_loss + - losses.correct_target_class_loss training_step_log: - loss @@ -136,8 +137,9 @@ model: x: "input_adv" seq030: loss: - - losses.hide_target_objects_loss - - losses.correct_target_class_loss + _call_with_args_: + - losses.hide_target_objects_loss + - losses.correct_target_class_loss test_sequence: seq005: perturbation @@ -147,5 +149,6 @@ model: x: "input_adv" seq030: loss: - - losses.hide_target_objects_loss - - losses.correct_target_class_loss + _call_with_args_: + - losses.hide_target_objects_loss + - losses.correct_target_class_loss diff --git a/mart/configs/model/yolov4.yaml b/mart/configs/model/yolov4.yaml index 7122b2d3..502b8f02 100644 --- a/mart/configs/model/yolov4.yaml +++ b/mart/configs/model/yolov4.yaml @@ -43,7 +43,8 @@ training_sequence: seq030: loss: - - losses.total_loss + _call_with_args_: + - losses.total_loss seq040: detections: @@ -79,7 +80,8 @@ validation_sequence: seq030: loss: - - losses.total_loss + _call_with_args_: + - losses.total_loss seq040: detections: @@ -117,7 +119,8 @@ test_sequence: seq030: loss: - - losses.total_loss + _call_with_args_: + - losses.total_loss seq040: detections: From f4cc7d65bd374c95d177870ca6d3cf29a30c7c66 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Thu, 15 Jun 2023 17:33:26 -0700 Subject: [PATCH 05/23] Reduce batch size --- mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml index b37e34f1..5f1988e8 100644 --- a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -51,8 +51,8 @@ callbacks: frequency: 500 datamodule: - num_workers: 32 - ims_per_batch: 16 + num_workers: 4 + ims_per_batch: 8 train_dataset: annFile: ${paths.data_dir}/coco/annotations/person_instances_train2017.json From 35ac996da5ae57bea384cf0f95437558379173a5 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Thu, 15 Jun 2023 17:33:32 -0700 Subject: [PATCH 06/23] Use logits --- mart/configs/model/yolov4.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mart/configs/model/yolov4.yaml b/mart/configs/model/yolov4.yaml index 502b8f02..6478e65f 100644 --- a/mart/configs/model/yolov4.yaml +++ b/mart/configs/model/yolov4.yaml @@ -48,7 +48,7 @@ training_sequence: seq040: detections: - preds: yolov4.preds + preds: yolov4.logits target: target seq050: @@ -85,7 +85,7 @@ validation_sequence: seq040: detections: - preds: yolov4.preds + preds: yolov4.logits target: target seq050: @@ -124,7 +124,7 @@ test_sequence: seq040: detections: - preds: yolov4.preds + preds: yolov4.logits target: target seq050: From 781d70ef9a4f7573c5ac11582f3b57cfdae61280 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 16 Jun 2023 12:45:53 -0700 Subject: [PATCH 07/23] Add YOLOv4 collate function --- mart/datamodules/coco.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/mart/datamodules/coco.py b/mart/datamodules/coco.py index 795f492d..538cf200 100644 --- a/mart/datamodules/coco.py +++ b/mart/datamodules/coco.py @@ -131,3 +131,29 @@ def yolo_collate_fn(batch): target[key] = default_collate(target[key]) return images, target + +def yolov4_collate_fn(batch): + images, targets = collate_fn(batch) + + images = default_collate(images) + + # Turn tuple of dicts into dict of tuples + keys = targets[0].keys() + target = {k: tuple(t[k] for t in targets) for k in keys} + + # Concatenate labels together along batch axis + packed = target["packed"] + packed = torch.cat(packed, dim=0) + + # Generate image indexs for labels along batch axis + packed_length = target["packed_length"] + indices = [i + torch.zeros((length,)) for i, length in enumerate(packed_length)] + indices = torch.cat(indices, dim=0)[..., None] + + # Concatenate indices with labels + target["packed"] = torch.cat([indices, packed], dim=-1) + + # Collate perturbable_mask + target["perturbable_mask"] = default_collate(target["perturbable_mask"]) + + return images, target From 76da651821fd862b36a965e0a0b70a70a81f8f5b Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 16 Jun 2023 12:46:07 -0700 Subject: [PATCH 08/23] Only compute metrics if we have a metric --- mart/models/modular.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mart/models/modular.py b/mart/models/modular.py index 76e728c7..48475312 100644 --- a/mart/models/modular.py +++ b/mart/models/modular.py @@ -170,12 +170,16 @@ def validation_step(self, batch, batch_idx): return output def validation_step_end(self, output): - self.validation_metrics(output["preds"], output["target"]) + if self.validation_metrics is not None: + self.validation_metrics(output["preds"], output["target"]) # I don't know why this is required to prevent CUDA memory leak in validaiton and test. (Not required in training.) output.clear() def validation_epoch_end(self, outputs): + if self.validation_metrics is None: + return + metrics = self.validation_metrics.compute() metrics = self.flatten_metrics(metrics) self.validation_metrics.reset() @@ -196,12 +200,16 @@ def test_step(self, batch, batch_idx): return output def test_step_end(self, output): - self.test_metrics(output["preds"], output["target"]) + if self.test_metrics is not None: + self.test_metrics(output["preds"], output["target"]) # I don't know why this is required to prevent CUDA memory leak in validaiton and test. (Not required in training.) output.clear() def test_epoch_end(self, outputs): + if self.test_metrics is None: + return + metrics = self.test_metrics.compute() metrics = self.flatten_metrics(metrics) self.test_metrics.reset() From ff8fdfdf98b72b8f6695f733bf6a72e3d088efcc Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 16 Jun 2023 12:46:52 -0700 Subject: [PATCH 09/23] ConvertBoxesToCXCYHW -> ConvertBoxesToCXCYWH --- mart/transforms/extended.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mart/transforms/extended.py b/mart/transforms/extended.py index 87f13ae0..9ec831a8 100644 --- a/mart/transforms/extended.py +++ b/mart/transforms/extended.py @@ -34,7 +34,7 @@ "ConvertCocoPolysToMask", "PadToSquare", "Resize", - "ConvertBoxesToCXCYHW", + "ConvertBoxesToCXCYWH", "RemapLabels", "PackBoxesAndLabels", "CreatePerturbableMaskFromImage", @@ -340,7 +340,7 @@ def resize_coordinates(self, coordinates, ratio): return coordinates -class ConvertBoxesToCXCYHW(ExTransform): +class ConvertBoxesToCXCYWH(ExTransform): def __call__( self, image: Tensor, @@ -349,7 +349,7 @@ def __call__( # X1Y1X2Y2 boxes = target["boxes"] - # X2Y2 -> HW + # X2Y2 -> WH boxes[:, 2] -= boxes[:, 0] boxes[:, 3] -= boxes[:, 1] From c70a3544ecd08139d8648f497805b286c47014a2 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 16 Jun 2023 12:47:28 -0700 Subject: [PATCH 10/23] Add normalize option to ConvertBoxesToCXCYWH --- mart/transforms/extended.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mart/transforms/extended.py b/mart/transforms/extended.py index 9ec831a8..7be14ea0 100644 --- a/mart/transforms/extended.py +++ b/mart/transforms/extended.py @@ -341,6 +341,11 @@ def resize_coordinates(self, coordinates, ratio): class ConvertBoxesToCXCYWH(ExTransform): + def __init__(self, normalize: bool = False): + super().__init__() + + self.normalize = normalize + def __call__( self, image: Tensor, @@ -357,6 +362,14 @@ def __call__( boxes[:, 0] += boxes[:, 2] / 2 boxes[:, 1] += boxes[:, 3] / 2 + if self.normalize: + # image.shape = CHW + C, H, W = image.shape + boxes[:, 0] /= W # x + boxes[:, 1] /= H # y + boxes[:, 2] /= W # w + boxes[:, 3] /= H # h + target["boxes"] = boxes return image, target From 3e3535f1f0b62c61f991d8afa09e83bc5dc4664c Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 16 Jun 2023 12:47:57 -0700 Subject: [PATCH 11/23] Add ability to disable one hot encoding and change order of packed labels in PackBoxesAndLabels --- mart/transforms/extended.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/mart/transforms/extended.py b/mart/transforms/extended.py index 7be14ea0..4763ce63 100644 --- a/mart/transforms/extended.py +++ b/mart/transforms/extended.py @@ -485,8 +485,13 @@ def __call__( class PackBoxesAndLabels(ExTransform): - def __init__(self, num_classes: int): - self.num_classes = num_classes + def __init__( + self, + num_classes: int | None, + order: list[str] = None, + ): + self.num_classes = num_classes or 0 + self.order = order or ["boxes", "scores", "labels"] def __call__( self, @@ -497,10 +502,17 @@ def __call__( labels = target["labels"] scores = torch.ones_like(labels)[..., None] - labels = torch.nn.functional.one_hot(labels, num_classes=self.num_classes) + if self.num_classes > 0: + labels = torch.nn.functional.one_hot(labels, num_classes=self.num_classes) + else: + labels = labels[..., None] + + # Concatenate/pack boxes, scores, and labels using given order. + values = { "boxes": boxes, "scores": scores, "labels": labels } + packed = torch.cat([values[key] for key in self.order], dim=-1) - target["packed"] = torch.cat([boxes, scores, labels], dim=-1) - target["packed_length"] = target["packed"].shape[0] + target["packed"] = packed + target["packed_length"] = packed.shape[0] return image, target From 98212ffca343797e4c15e9573b8d7d6ff732b3ba Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 16 Jun 2023 12:49:06 -0700 Subject: [PATCH 12/23] Add coco_yolov4 datamodule --- mart/configs/datamodule/coco_yolov4.yaml | 31 +++++++++++++++++++ .../experiment/COCO_YOLOv4_ShapeShifter.yaml | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 mart/configs/datamodule/coco_yolov4.yaml diff --git a/mart/configs/datamodule/coco_yolov4.yaml b/mart/configs/datamodule/coco_yolov4.yaml new file mode 100644 index 00000000..3e63a2b9 --- /dev/null +++ b/mart/configs/datamodule/coco_yolov4.yaml @@ -0,0 +1,31 @@ +defaults: + - coco + +num_workers: 1 + +train_dataset: + transforms: + transforms: + - _target_: torchvision.transforms.ToTensor + - _target_: mart.transforms.ConvertCocoPolysToMask + - _target_: mart.transforms.PadToSquare + fill: 0.5 + - _target_: mart.transforms.Resize + size: [416, 416] + - _target_: mart.transforms.ConvertBoxesToCXCYWH + normalize: True + - _target_: mart.transforms.RemapLabels + - _target_: mart.transforms.PackBoxesAndLabels + num_classes: null # don't one-hot encode labels + order: ["labels", "boxes"] # abuse scores for indexes + - _target_: mart.transforms.ConvertInstanceSegmentationToPerturbable + +val_dataset: + transforms: ${..train_dataset.transforms} + +test_dataset: + transforms: ${..val_dataset.transforms} + +collate_fn: + _target_: hydra.utils.get_method + path: mart.datamodules.coco.yolov4_collate_fn diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml index 5f1988e8..55d18ce5 100644 --- a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -7,7 +7,7 @@ defaults: - /attack/composer@model.modules.input_adv: warp_composite - /attack/gradient_modifier@model.gradient_modifier: lp_normalizer - override /optimization: super_convergence - - override /datamodule: coco_yolov3 + - override /datamodule: coco_yolov4 - override /model: yolov4 - override /metric: average_precision - override /callbacks: From ebf244a0afdbb4257915c3e7ad7926a642272801 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 16 Jun 2023 12:53:24 -0700 Subject: [PATCH 13/23] Update sequences --- .../experiment/COCO_YOLOv4_ShapeShifter.yaml | 40 +----- mart/configs/model/yolov4.yaml | 134 ++---------------- 2 files changed, 18 insertions(+), 156 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml index 55d18ce5..dde058c9 100644 --- a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -91,43 +91,23 @@ model: interpolation: 2 # BILINEAR clamp: [0, 1] - loss: - weights: [1, 1] - optimizer: lr: 0.01 momentum: 0.9 + maximize: True gradient_modifier: null + training_metrics: null + validation_metrics: null + test_metrics: null + training_sequence: seq005: perturbation seq006: input_adv seq010: yolov4: x: "input_adv" - seq030: - loss: - _call_with_args_: - - losses.hide_target_objects_loss - - losses.correct_target_class_loss - - training_step_log: - - loss - - total_loss - - coord_loss - - obj_loss - - noobj_loss - - class_loss - - hide_objects_loss - - target_class_loss - - hide_target_objects_loss - - correct_target_class_loss - - target_count - - score_count - - target_score_count - - training_metrics: null validation_sequence: seq005: perturbation @@ -135,11 +115,6 @@ model: seq010: yolov4: x: "input_adv" - seq030: - loss: - _call_with_args_: - - losses.hide_target_objects_loss - - losses.correct_target_class_loss test_sequence: seq005: perturbation @@ -147,8 +122,3 @@ model: seq010: yolov4: x: "input_adv" - seq030: - loss: - _call_with_args_: - - losses.hide_target_objects_loss - - losses.correct_target_class_loss diff --git a/mart/configs/model/yolov4.yaml b/mart/configs/model/yolov4.yaml index 6478e65f..2d7645a2 100644 --- a/mart/configs/model/yolov4.yaml +++ b/mart/configs/model/yolov4.yaml @@ -6,24 +6,13 @@ modules: # XXX: This is so sketchy... _target_: torch.hub.load repo_or_dir: "AlexeyAB/Yet-Another-YOLOv4-Pytorch" + trust_repo: False model: "yolov4" pretrained: True - trust_repo: False - - losses: - _target_: mart.models.yolov3.Loss - image_size: 416 # FIXME: use ${training_data.transform.image_size}? - average: True loss: _target_: mart.nn.Sum - detections: - _target_: mart.models.yolov3.Detections - nms: true - conf_thres: 0.1 - nms_thres: 0.4 - output: _target_: mart.nn.ReturnKwargs @@ -35,157 +24,60 @@ training_sequence: yolov4: _return_as_dict_: ["logits", "loss"] x: "input" - - seq020: - losses: - logits: yolov4.logits - target: target + y: "target.packed" seq030: loss: _call_with_args_: - - losses.total_loss - - seq040: - detections: - preds: yolov4.logits - target: target + - yolov4.loss seq050: output: loss: loss - total_loss: losses.total_loss - coord_loss: losses.coord_loss - obj_loss: losses.obj_loss - noobj_loss: losses.noobj_loss - class_loss: losses.class_loss - hide_objects_loss: losses.hide_objects_loss - target_class_loss: losses.target_class_loss - hide_target_objects_loss: losses.hide_target_objects_loss - correct_target_class_loss: losses.correct_target_class_loss - target_count: losses.target_count - score_count: losses.score_count - target_score_count: losses.target_score_count + +# validation sequence does not produce preds/targets +validation_metrics: null validation_sequence: seq010: yolov4: _return_as_dict_: ["logits", "loss"] x: "input" - - seq020: - losses: - logits: yolov4.logits - target: target + y: "target.packed" seq030: loss: _call_with_args_: - - losses.total_loss - - seq040: - detections: - preds: yolov4.logits - target: target + - yolov4.loss seq050: output: - preds: detections.preds - target: detections.targets loss: loss - total_loss: losses.total_loss - coord_loss: losses.coord_loss - obj_loss: losses.obj_loss - noobj_loss: losses.noobj_loss - class_loss: losses.class_loss - hide_objects_loss: losses.hide_objects_loss - target_class_loss: losses.target_class_loss - hide_target_objects_loss: losses.hide_target_objects_loss - correct_target_class_loss: losses.correct_target_class_loss - target_count: losses.target_count - score_count: losses.score_count - target_score_count: losses.target_score_count + +# test sequence does not produce preds/targets +test_metrics: null test_sequence: seq010: yolov4: _return_as_dict_: ["logits", "loss"] x: "input" - - seq020: - losses: - logits: yolov4.logits - target: target + y: "target.packed" seq030: loss: _call_with_args_: - - losses.total_loss - - seq040: - detections: - preds: yolov4.logits - target: target + - yolov4.loss seq050: output: - preds: detections.preds - target: detections.targets loss: loss - total_loss: losses.total_loss - coord_loss: losses.coord_loss - obj_loss: losses.obj_loss - noobj_loss: losses.noobj_loss - class_loss: losses.class_loss - hide_objects_loss: losses.hide_objects_loss - target_class_loss: losses.target_class_loss - hide_target_objects_loss: losses.hide_target_objects_loss - correct_target_class_loss: losses.correct_target_class_loss - target_count: losses.target_count - score_count: losses.score_count - target_score_count: losses.target_score_count training_step_log: - loss - - total_loss - - coord_loss - - obj_loss - - noobj_loss - - class_loss - - hide_objects_loss - - target_class_loss - - hide_target_objects_loss - - correct_target_class_loss - - target_count - - score_count - - target_score_count validation_step_log: - loss - - total_loss - - coord_loss - - obj_loss - - noobj_loss - - class_loss - - hide_objects_loss - - target_class_loss - - hide_target_objects_loss - - correct_target_class_loss - - target_count - - score_count - - target_score_count test_step_log: - loss - - total_loss - - coord_loss - - obj_loss - - noobj_loss - - class_loss - - hide_objects_loss - - target_class_loss - - hide_target_objects_loss - - correct_target_class_loss - - target_count - - score_count - - target_score_count From c7602ae667f42c408ab35d730237bf6fc389ab70 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 16 Jun 2023 13:37:45 -0700 Subject: [PATCH 14/23] Remove model checkpoint callback --- mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml index dde058c9..d2eabde8 100644 --- a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -12,7 +12,6 @@ defaults: - override /metric: average_precision - override /callbacks: [ - model_checkpoint, lr_monitor, perturbation_visualizer, gradient_monitor, @@ -34,10 +33,6 @@ trainer: precision: 32 callbacks: - model_checkpoint: - monitor: "validation_metrics/map" - mode: "min" - attack_in_eval_mode: module_classes: - _target_: hydra.utils.get_class From 41685e7995cfc8c87f9d5a4bf23bba44fc8e1fa9 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Fri, 16 Jun 2023 14:49:51 -0700 Subject: [PATCH 15/23] Make YOLOv3 losses work --- mart/configs/datamodule/coco_yolov3.yaml | 3 +- mart/configs/datamodule/coco_yolov4.yaml | 31 ------------------- .../experiment/COCO_YOLOv4_ShapeShifter.yaml | 8 +++-- mart/configs/model/yolov4.yaml | 30 +++++++++++++++--- mart/datamodules/coco.py | 30 +++++++++--------- 5 files changed, 47 insertions(+), 55 deletions(-) delete mode 100644 mart/configs/datamodule/coco_yolov4.yaml diff --git a/mart/configs/datamodule/coco_yolov3.yaml b/mart/configs/datamodule/coco_yolov3.yaml index 27be09d1..d9fd6326 100644 --- a/mart/configs/datamodule/coco_yolov3.yaml +++ b/mart/configs/datamodule/coco_yolov3.yaml @@ -12,7 +12,8 @@ train_dataset: fill: 0.5 - _target_: mart.transforms.Resize size: [416, 416] - - _target_: mart.transforms.ConvertBoxesToCXCYHW + - _target_: mart.transforms.ConvertBoxesToCXCYWH + normalize: True - _target_: mart.transforms.RemapLabels - _target_: mart.transforms.PackBoxesAndLabels num_classes: 80 diff --git a/mart/configs/datamodule/coco_yolov4.yaml b/mart/configs/datamodule/coco_yolov4.yaml deleted file mode 100644 index 3e63a2b9..00000000 --- a/mart/configs/datamodule/coco_yolov4.yaml +++ /dev/null @@ -1,31 +0,0 @@ -defaults: - - coco - -num_workers: 1 - -train_dataset: - transforms: - transforms: - - _target_: torchvision.transforms.ToTensor - - _target_: mart.transforms.ConvertCocoPolysToMask - - _target_: mart.transforms.PadToSquare - fill: 0.5 - - _target_: mart.transforms.Resize - size: [416, 416] - - _target_: mart.transforms.ConvertBoxesToCXCYWH - normalize: True - - _target_: mart.transforms.RemapLabels - - _target_: mart.transforms.PackBoxesAndLabels - num_classes: null # don't one-hot encode labels - order: ["labels", "boxes"] # abuse scores for indexes - - _target_: mart.transforms.ConvertInstanceSegmentationToPerturbable - -val_dataset: - transforms: ${..train_dataset.transforms} - -test_dataset: - transforms: ${..val_dataset.transforms} - -collate_fn: - _target_: hydra.utils.get_method - path: mart.datamodules.coco.yolov4_collate_fn diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml index d2eabde8..d3e8f78c 100644 --- a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -7,7 +7,7 @@ defaults: - /attack/composer@model.modules.input_adv: warp_composite - /attack/gradient_modifier@model.gradient_modifier: lp_normalizer - override /optimization: super_convergence - - override /datamodule: coco_yolov4 + - override /datamodule: coco_yolov3 - override /model: yolov4 - override /metric: average_precision - override /callbacks: @@ -56,6 +56,10 @@ datamodule: test_dataset: annFile: ${paths.data_dir}/coco/annotations/person_instances_val2017.json + collate_fn: + path: mart.datamodules.coco.yolov4_collate_fn + + model: modules: perturbation: @@ -87,7 +91,7 @@ model: clamp: [0, 1] optimizer: - lr: 0.01 + lr: 0.1 momentum: 0.9 maximize: True diff --git a/mart/configs/model/yolov4.yaml b/mart/configs/model/yolov4.yaml index 2d7645a2..da17b069 100644 --- a/mart/configs/model/yolov4.yaml +++ b/mart/configs/model/yolov4.yaml @@ -10,6 +10,11 @@ modules: model: "yolov4" pretrained: True + losses: + _target_: mart.models.yolov3.Loss + image_size: 416 # FIXME: use ${training_data.transform.image_size}? + average: True + loss: _target_: mart.nn.Sum @@ -24,16 +29,21 @@ training_sequence: yolov4: _return_as_dict_: ["logits", "loss"] x: "input" - y: "target.packed" + y: "target.packedv4" + + seq020: + losses: + logits: "yolov4.logits" + target: "target" seq030: loss: _call_with_args_: - - yolov4.loss + - "yolov4.loss" seq050: output: - loss: loss + loss: "loss" # validation sequence does not produce preds/targets validation_metrics: null @@ -43,7 +53,12 @@ validation_sequence: yolov4: _return_as_dict_: ["logits", "loss"] x: "input" - y: "target.packed" + y: "target.packedv4" + + seq020: + losses: + logits: "yolov4.logits" + target: "target" seq030: loss: @@ -62,7 +77,12 @@ test_sequence: yolov4: _return_as_dict_: ["logits", "loss"] x: "input" - y: "target.packed" + y: "target.packedv4" + + seq020: + losses: + logits: "yolov4.logits" + target: "target" seq030: loss: diff --git a/mart/datamodules/coco.py b/mart/datamodules/coco.py index 538cf200..0c3095ec 100644 --- a/mart/datamodules/coco.py +++ b/mart/datamodules/coco.py @@ -113,7 +113,7 @@ def to_padded_tensor(tensors, dim=0, fill=0.0): def yolo_collate_fn(batch): - images, targets = tuple(zip(*batch)) + images, targets = collate_fn(batch) images = default_collate(images) @@ -132,28 +132,26 @@ def yolo_collate_fn(batch): return images, target +# FIXME: Turn this into a class with options def yolov4_collate_fn(batch): - images, targets = collate_fn(batch) - - images = default_collate(images) - - # Turn tuple of dicts into dict of tuples - keys = targets[0].keys() - target = {k: tuple(t[k] for t in targets) for k in keys} - - # Concatenate labels together along batch axis - packed = target["packed"] - packed = torch.cat(packed, dim=0) + images, target = yolo_collate_fn(batch) # Generate image indexs for labels along batch axis packed_length = target["packed_length"] indices = [i + torch.zeros((length,)) for i, length in enumerate(packed_length)] indices = torch.cat(indices, dim=0)[..., None] - # Concatenate indices with labels - target["packed"] = torch.cat([indices, packed], dim=-1) + # Concatenate labels together along batch axis + packed = [target["packed"][i][:length] for i, length in enumerate(packed_length)] + packed = torch.cat(packed, dim=0) + + # boxes, scores, labels -> labels, boxes + boxes = packed[:, 0:4] + scores = packed[:, 4:5] + labels = torch.argmax(packed[:, 5:], dim=-1, keepdim=True) - # Collate perturbable_mask - target["perturbable_mask"] = default_collate(target["perturbable_mask"]) + # concatenate in correct order + packed = torch.cat([indices, labels, boxes], dim=-1) + target["packedv4"] = packed return images, target From e538aa60ec0fe6881256ffbc9630e19d0ab2842a Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Tue, 20 Jun 2023 09:40:37 -0700 Subject: [PATCH 16/23] Revert mart/configs/datamodule/coco_yolov3.yaml to shapeshifter_yolov3 --- mart/configs/datamodule/coco_yolov3.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mart/configs/datamodule/coco_yolov3.yaml b/mart/configs/datamodule/coco_yolov3.yaml index d9fd6326..27be09d1 100644 --- a/mart/configs/datamodule/coco_yolov3.yaml +++ b/mart/configs/datamodule/coco_yolov3.yaml @@ -12,8 +12,7 @@ train_dataset: fill: 0.5 - _target_: mart.transforms.Resize size: [416, 416] - - _target_: mart.transforms.ConvertBoxesToCXCYWH - normalize: True + - _target_: mart.transforms.ConvertBoxesToCXCYHW - _target_: mart.transforms.RemapLabels - _target_: mart.transforms.PackBoxesAndLabels num_classes: 80 From 5782d94682d55fc181e804556a83809066d7aa67 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Tue, 20 Jun 2023 09:47:39 -0700 Subject: [PATCH 17/23] bugfix --- mart/configs/datamodule/coco_yolov3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mart/configs/datamodule/coco_yolov3.yaml b/mart/configs/datamodule/coco_yolov3.yaml index 27be09d1..d8221803 100644 --- a/mart/configs/datamodule/coco_yolov3.yaml +++ b/mart/configs/datamodule/coco_yolov3.yaml @@ -12,7 +12,7 @@ train_dataset: fill: 0.5 - _target_: mart.transforms.Resize size: [416, 416] - - _target_: mart.transforms.ConvertBoxesToCXCYHW + - _target_: mart.transforms.ConvertBoxesToCXCYWH - _target_: mart.transforms.RemapLabels - _target_: mart.transforms.PackBoxesAndLabels num_classes: 80 From 83b48fffc7d6a22aba43d621635b58f0a017c616 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Tue, 20 Jun 2023 15:18:16 -0700 Subject: [PATCH 18/23] Fix config --- mart/configs/model/yolov4.yaml | 83 +++++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/mart/configs/model/yolov4.yaml b/mart/configs/model/yolov4.yaml index da17b069..98a3a510 100644 --- a/mart/configs/model/yolov4.yaml +++ b/mart/configs/model/yolov4.yaml @@ -18,6 +18,12 @@ modules: loss: _target_: mart.nn.Sum + detections: + _target_: mart.models.yolov3.Detections + nms: true + conf_thres: 0.1 + nms_thres: 0.4 + output: _target_: mart.nn.ReturnKwargs @@ -27,9 +33,7 @@ training_metrics: null training_sequence: seq010: yolov4: - _return_as_dict_: ["logits", "loss"] x: "input" - y: "target.packedv4" seq020: losses: @@ -39,21 +43,33 @@ training_sequence: seq030: loss: _call_with_args_: - - "yolov4.loss" + - "losses.total_loss" + + seq040: + detections: + preds: "yolov4.preds" + target: "target" seq050: output: loss: "loss" - -# validation sequence does not produce preds/targets -validation_metrics: null + total_loss: "losses.total_loss" + coord_loss: "losses.coord_loss" + obj_loss: "losses.obj_loss" + noobj_loss: "losses.noobj_loss" + class_loss: "losses.class_loss" + hide_objects_loss: "losses.hide_objects_loss" + target_class_loss: "losses.target_class_loss" + hide_target_objects_loss: "losses.hide_target_objects_loss" + correct_target_class_loss: "losses.correct_target_class_loss" + target_count: "losses.target_count" + score_count: "losses.score_count" + target_score_count: "losses.target_score_count" validation_sequence: seq010: yolov4: - _return_as_dict_: ["logits", "loss"] x: "input" - y: "target.packedv4" seq020: losses: @@ -63,21 +79,35 @@ validation_sequence: seq030: loss: _call_with_args_: - - yolov4.loss + - "losses.total_loss" + + seq040: + detections: + preds: "yolov4.preds" + target: "target" seq050: output: - loss: loss - -# test sequence does not produce preds/targets -test_metrics: null + preds: "detections.preds" + target: "detections.targets" + loss: "loss" + total_loss: "losses.total_loss" + coord_loss: "losses.coord_loss" + obj_loss: "losses.obj_loss" + noobj_loss: "losses.noobj_loss" + class_loss: "losses.class_loss" + hide_objects_loss: "losses.hide_objects_loss" + target_class_loss: "losses.target_class_loss" + hide_target_objects_loss: "losses.hide_target_objects_loss" + correct_target_class_loss: "losses.correct_target_class_loss" + target_count: "losses.target_count" + score_count: "losses.score_count" + target_score_count: "losses.target_score_count" test_sequence: seq010: yolov4: - _return_as_dict_: ["logits", "loss"] x: "input" - y: "target.packedv4" seq020: losses: @@ -87,11 +117,30 @@ test_sequence: seq030: loss: _call_with_args_: - - yolov4.loss + - "losses.total_loss" + + seq040: + detections: + preds: "yolov4.preds" + target: "target" seq050: output: - loss: loss + preds: "detections.preds" + target: "detections.targets" + loss: "loss" + total_loss: "losses.total_loss" + coord_loss: "losses.coord_loss" + obj_loss: "losses.obj_loss" + noobj_loss: "losses.noobj_loss" + class_loss: "losses.class_loss" + hide_objects_loss: "losses.hide_objects_loss" + target_class_loss: "losses.target_class_loss" + hide_target_objects_loss: "losses.hide_target_objects_loss" + correct_target_class_loss: "losses.correct_target_class_loss" + target_count: "losses.target_count" + score_count: "losses.score_count" + target_score_count: "losses.target_score_count" training_step_log: - loss From b67fa5dbfd288dc5e00abbe1ed2d5afa000f8e3c Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Tue, 20 Jun 2023 15:56:35 -0700 Subject: [PATCH 19/23] Update YOLOv4 ShapeShifter attack --- .../experiment/COCO_YOLOv4_ShapeShifter.yaml | 41 +++++++++++++++---- mart/configs/model/yolov4.yaml | 38 +++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml index d3e8f78c..c1e25192 100644 --- a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -12,6 +12,7 @@ defaults: - override /metric: average_precision - override /callbacks: [ + model_checkpoint, lr_monitor, perturbation_visualizer, gradient_monitor, @@ -33,6 +34,10 @@ trainer: precision: 32 callbacks: + model_checkpoint: + monitor: "validation_metrics/map" + mode: "min" + attack_in_eval_mode: module_classes: - _target_: hydra.utils.get_class @@ -46,7 +51,7 @@ callbacks: frequency: 500 datamodule: - num_workers: 4 + num_workers: 16 ims_per_batch: 8 train_dataset: @@ -59,7 +64,6 @@ datamodule: collate_fn: path: mart.datamodules.coco.yolov4_collate_fn - model: modules: perturbation: @@ -91,22 +95,27 @@ model: clamp: [0, 1] optimizer: - lr: 0.1 + lr: 0.02 momentum: 0.9 - maximize: True gradient_modifier: null - training_metrics: null - validation_metrics: null - test_metrics: null - training_sequence: seq005: perturbation seq006: input_adv seq010: yolov4: x: "input_adv" + seq030: + loss: + _call_with_args_: + - losses.hide_target_objects_loss + - losses.correct_target_class_loss + weights: + - 1 + - 1 + + training_metrics: null validation_sequence: seq005: perturbation @@ -114,6 +123,14 @@ model: seq010: yolov4: x: "input_adv" + seq030: + loss: + _call_with_args_: + - losses.hide_target_objects_loss + - losses.correct_target_class_loss + weights: + - 1 + - 1 test_sequence: seq005: perturbation @@ -121,3 +138,11 @@ model: seq010: yolov4: x: "input_adv" + seq030: + loss: + _call_with_args_: + - losses.hide_target_objects_loss + - losses.correct_target_class_loss + weights: + - 1 + - 1 diff --git a/mart/configs/model/yolov4.yaml b/mart/configs/model/yolov4.yaml index 98a3a510..8e13f39c 100644 --- a/mart/configs/model/yolov4.yaml +++ b/mart/configs/model/yolov4.yaml @@ -144,9 +144,47 @@ test_sequence: training_step_log: - loss + - total_loss + - coord_loss + - obj_loss + - noobj_loss + - class_loss + - hide_objects_loss + - target_class_loss + - hide_target_objects_loss + - correct_target_class_loss + - target_count + - score_count + - target_score_count + validation_step_log: - loss + - total_loss + - coord_loss + - obj_loss + - noobj_loss + - class_loss + - hide_objects_loss + - target_class_loss + - hide_target_objects_loss + - correct_target_class_loss + - target_count + - score_count + - target_score_count + test_step_log: - loss + - total_loss + - coord_loss + - obj_loss + - noobj_loss + - class_loss + - hide_objects_loss + - target_class_loss + - hide_target_objects_loss + - correct_target_class_loss + - target_count + - score_count + - target_score_count From a9c490ccb2a71c0c13f3039906f7823eed7fe713 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Wed, 21 Jun 2023 08:02:42 -0700 Subject: [PATCH 20/23] Add total variation loss --- mart/models/yolov3.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/mart/models/yolov3.py b/mart/models/yolov3.py index 38ed9db7..15f08837 100644 --- a/mart/models/yolov3.py +++ b/mart/models/yolov3.py @@ -101,7 +101,7 @@ def __init__(self, image_size, average=True, score_thresh=0.01, target_idx=0): self.score_thresh = score_thresh self.target_idx = target_idx - def forward(self, logits, target, **kwargs): + def forward(self, logits, target, perturbation=None, **kwargs): targets = target["packed"] lengths = target["packed_length"] @@ -149,6 +149,19 @@ def forward(self, logits, target, **kwargs): target_count = target_mask.sum() / logits.shape[0] target_score_count = (target_mask & score_mask).sum() / logits.shape[0] + total_variation = None + if perturbation is not None: + # FIXME: This is a hack + total_variation = torch.mean( + torch.sum( + torch.square(perturbation[:, 1:, :] - perturbation[:, :-1, :]) + ) + + torch.sum( # noqa: W503 + torch.square(perturbation[:, :, 1:] - perturbation[:, :, :-1]) + ) + ) + + return { "total_loss": total_loss, "coord_loss": coord_loss, @@ -162,6 +175,7 @@ def forward(self, logits, target, **kwargs): "score_count": score_count, "target_count": target_count, "target_score_count": target_score_count, + "total_variation": total_variation, } From 098eb2d4502995f98b8a96f959305cdca98494cd Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Wed, 21 Jun 2023 08:03:34 -0700 Subject: [PATCH 21/23] Change losses and add total variation --- .../experiment/COCO_YOLOv4_ShapeShifter.yaml | 63 ++++++++++++++----- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml index c1e25192..bf2d7289 100644 --- a/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml +++ b/mart/configs/experiment/COCO_YOLOv4_ShapeShifter.yaml @@ -95,54 +95,85 @@ model: clamp: [0, 1] optimizer: - lr: 0.02 + lr: 0.05 # ims_per_batch / orig_ims_per_batch * orig_lr = (8 / 16) * 0.1 momentum: 0.9 + lr_scheduler: + scheduler: + three_phase: true + gradient_modifier: null training_sequence: - seq005: perturbation - seq006: input_adv + seq005: "perturbation" + seq006: "input_adv" seq010: yolov4: x: "input_adv" + seq020: + losses: + perturbation: "perturbation" seq030: loss: _call_with_args_: - - losses.hide_target_objects_loss - - losses.correct_target_class_loss + - "losses.hide_objects_loss" + - "losses.total_variation" weights: - 1 - - 1 + - 0.0001 + seq050: + output: + total_variation: "losses.total_variation" training_metrics: null + training_step_log: + - loss + - total_loss + - coord_loss + - obj_loss + - noobj_loss + - class_loss + - hide_objects_loss + - target_class_loss + - hide_target_objects_loss + - correct_target_class_loss + - target_count + - score_count + - target_score_count + - total_variation validation_sequence: - seq005: perturbation - seq006: input_adv + seq005: "perturbation" + seq006: "input_adv" seq010: yolov4: x: "input_adv" + seq020: + losses: + perturbation: "perturbation" seq030: loss: _call_with_args_: - - losses.hide_target_objects_loss - - losses.correct_target_class_loss + - "losses.hide_objects_loss" + - "losses.total_variation" weights: - 1 - - 1 + - 0.0001 test_sequence: - seq005: perturbation - seq006: input_adv + seq005: "perturbation" + seq006: "input_adv" seq010: yolov4: x: "input_adv" + seq020: + losses: + perturbation: "perturbation" seq030: loss: _call_with_args_: - - losses.hide_target_objects_loss - - losses.correct_target_class_loss + - "losses.hide_objects_loss" + - "losses.total_variation" weights: - 1 - - 1 + - 0.0001 From 97ae58dffa5676fd7517e77bd0f4112ee4142402 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Wed, 21 Jun 2023 08:04:05 -0700 Subject: [PATCH 22/23] Add COCO_YOLOv4 experiment --- mart/configs/experiment/COCO_YOLOv4.yaml | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 mart/configs/experiment/COCO_YOLOv4.yaml diff --git a/mart/configs/experiment/COCO_YOLOv4.yaml b/mart/configs/experiment/COCO_YOLOv4.yaml new file mode 100644 index 00000000..b0abd4ab --- /dev/null +++ b/mart/configs/experiment/COCO_YOLOv4.yaml @@ -0,0 +1,33 @@ +# @package _global_ + +defaults: + - override /datamodule: coco_yolov3 + - override /model: yolov4 + - override /optimization: super_convergence + - override /metric: average_precision + +task_name: "COCO_YOLOv4" +tags: ["evaluation"] + +optimized_metric: "test_metrics/map" + +trainer: + # 117,266 training images, 6 epochs, batch_size=16, 43,974.75 + max_steps: 43975 + # FIXME: "nms_kernel" not implemented for 'BFloat16', torch.ops.torchvision.nms(). + precision: 32 + +datamodule: + num_workers: 32 + ims_per_batch: 16 + + collate_fn: + path: mart.datamodules.coco.yolov4_collate_fn + +model: + optimizer: + lr: 0.001 + momentum: 0.9 + weight_decay: 0.0005 + + training_metrics: null From ec6145af3bb0a1dad770896016f3861d551b1de3 Mon Sep 17 00:00:00 2001 From: Cory Cornelius Date: Wed, 21 Jun 2023 17:15:09 -0700 Subject: [PATCH 23/23] style --- mart/configs/model/yolov4.yaml | 2 -- mart/datamodules/coco.py | 3 ++- mart/models/yolov3.py | 6 +----- mart/nn/nn.py | 12 +++++++++--- mart/transforms/extended.py | 10 +++++----- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/mart/configs/model/yolov4.yaml b/mart/configs/model/yolov4.yaml index 8e13f39c..c8f16e2a 100644 --- a/mart/configs/model/yolov4.yaml +++ b/mart/configs/model/yolov4.yaml @@ -157,7 +157,6 @@ training_step_log: - score_count - target_score_count - validation_step_log: - loss - total_loss @@ -173,7 +172,6 @@ validation_step_log: - score_count - target_score_count - test_step_log: - loss - total_loss diff --git a/mart/datamodules/coco.py b/mart/datamodules/coco.py index 0c3095ec..1e868841 100644 --- a/mart/datamodules/coco.py +++ b/mart/datamodules/coco.py @@ -132,11 +132,12 @@ def yolo_collate_fn(batch): return images, target + # FIXME: Turn this into a class with options def yolov4_collate_fn(batch): images, target = yolo_collate_fn(batch) - # Generate image indexs for labels along batch axis + # Generate image indexes for labels along batch axis packed_length = target["packed_length"] indices = [i + torch.zeros((length,)) for i, length in enumerate(packed_length)] indices = torch.cat(indices, dim=0)[..., None] diff --git a/mart/models/yolov3.py b/mart/models/yolov3.py index 15f08837..8a4556b4 100644 --- a/mart/models/yolov3.py +++ b/mart/models/yolov3.py @@ -151,17 +151,13 @@ def forward(self, logits, target, perturbation=None, **kwargs): total_variation = None if perturbation is not None: - # FIXME: This is a hack total_variation = torch.mean( - torch.sum( - torch.square(perturbation[:, 1:, :] - perturbation[:, :-1, :]) - ) + torch.sum(torch.square(perturbation[:, 1:, :] - perturbation[:, :-1, :])) + torch.sum( # noqa: W503 torch.square(perturbation[:, :, 1:] - perturbation[:, :, :-1]) ) ) - return { "total_loss": total_loss, "coord_loss": coord_loss, diff --git a/mart/nn/nn.py b/mart/nn/nn.py index 6dd21905..e1ddc5d6 100644 --- a/mart/nn/nn.py +++ b/mart/nn/nn.py @@ -165,7 +165,10 @@ def __call__( # Append kwargs to args using arg_keys try: - [args.append(kwargs[kwargs_key] if isinstance(kwargs_key, str) else kwargs_key) for kwargs_key in arg_keys] + [ + args.append(kwargs[kwargs_key] if isinstance(kwargs_key, str) else kwargs_key) + for kwargs_key in arg_keys + ] except KeyError as ex: raise Exception( f"{module_name} only received kwargs: {', '.join(kwargs.keys())}." @@ -173,7 +176,10 @@ def __call__( # Replace kwargs with selected kwargs try: - kwargs = {name: kwargs[kwargs_key] if isinstance(kwargs_key, str) else kwargs_key for name, kwargs_key in kwarg_keys.items()} + kwargs = { + name: kwargs[kwargs_key] if isinstance(kwargs_key, str) else kwargs_key + for name, kwargs_key in kwarg_keys.items() + } except KeyError as ex: raise Exception( f"{module_name} only received kwargs: {', '.join(kwargs.keys())}." @@ -240,7 +246,7 @@ def __getitem__(self, key): elif isinstance(value, dict) and subkey in value: value = value[subkey] else: - raise KeyError("No {subkey} in " + ".".join([key, *subkeys])) + raise KeyError(f"No {subkey} in " + ".".join([key, *subkeys])) return value diff --git a/mart/transforms/extended.py b/mart/transforms/extended.py index 4763ce63..8a04932b 100644 --- a/mart/transforms/extended.py +++ b/mart/transforms/extended.py @@ -365,10 +365,10 @@ def __call__( if self.normalize: # image.shape = CHW C, H, W = image.shape - boxes[:, 0] /= W # x - boxes[:, 1] /= H # y - boxes[:, 2] /= W # w - boxes[:, 3] /= H # h + boxes[:, 0] /= W # x + boxes[:, 1] /= H # y + boxes[:, 2] /= W # w + boxes[:, 3] /= H # h target["boxes"] = boxes @@ -508,7 +508,7 @@ def __call__( labels = labels[..., None] # Concatenate/pack boxes, scores, and labels using given order. - values = { "boxes": boxes, "scores": scores, "labels": labels } + values = {"boxes": boxes, "scores": scores, "labels": labels} packed = torch.cat([values[key] for key in self.order], dim=-1) target["packed"] = packed