what.examples.yolov3_tog_attack_demo

  1import cv2
  2import numpy as np
  3
  4from what.models.detection.datasets.coco import COCO_CLASS_NAMES
  5from what.models.detection.utils.box_utils import draw_bounding_boxes
  6from what.models.detection.yolo.yolov3 import YOLOV3
  7from what.models.detection.yolo.utils.yolo_utils import yolo_process_output, yolov3_anchors, yolov3_tiny_anchors
  8
  9from what.attacks.detection.yolo.TOG import TOGAttack
 10from what.utils.resize import bilinear_resize
 11
 12from what.cli.model import *
 13from what.utils.file import get_file
 14
 15# Target Model
 16what_yolov3_model_list = what_model_list[0:4]
 17
 18def yolov3_tog_attack_demo():
 19
 20    classes = COCO_CLASS_NAMES
 21
 22    colors = np.random.uniform(0, 255, size=(len(classes), 3))
 23
 24    # Check what_model_list for all supported models
 25    index = 3
 26
 27    # Download the model first if not exists
 28    if not os.path.isfile(os.path.join(WHAT_MODEL_PATH, what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX])):
 29        get_file(what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX],
 30                    WHAT_MODEL_PATH,
 31                    what_yolov3_model_list[index][WHAT_MODEL_URL_INDEX],
 32                    what_yolov3_model_list[index][WHAT_MODEL_HASH_INDEX])
 33
 34    attack = TOGAttack(os.path.join(WHAT_MODEL_PATH, what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX]), classes)
 35    attack.fixed = False
 36
 37    last_outs = None
 38    last_boxes = None
 39    last_probs = None
 40
 41    # Initialize the camera
 42    video = input(f"Please input the OpenCV capture device (e.g. 0, 1, 2): ")
 43
 44    while not video.isdigit():
 45        video = input(f"Please input the OpenCV capture device (e.g. 0, 1, 2): ")
 46
 47    # Capture from camera
 48    cap = cv2.VideoCapture(int(video))
 49    #cap.set(3, 1920)
 50    #cap.set(4, 1080)
 51
 52    n = 0
 53
 54    while(True): 
 55        # Capture the video frame
 56        success, origin_cv_image = cap.read()  # read the camera frame
 57        if not success:
 58            break
 59
 60        # For YOLO, the input pixel values are normalized to [0, 1]
 61        input_cv_image = cv2.resize(origin_cv_image, (416, 416))
 62        input_cv_image = np.array(input_cv_image).astype(np.float32) / 255.0
 63
 64        # Image preprocessing
 65        input_cv_image = cv2.cvtColor(input_cv_image, cv2.COLOR_BGR2RGB)
 66
 67        # Yolo inference
 68        input_cv_image, outs = attack.attack(input_cv_image)
 69
 70        if last_outs is not None:
 71            res_list = []
 72            for out, last_out in zip(outs, last_outs):
 73                out = out.reshape((-1, 5 + len(classes)))
 74                last_out = last_out.reshape((-1, 5 + len(classes)))
 75
 76                res = np.mean(out[:, 4] - last_out[:, 4])
 77                res_list.append(res)
 78        else:
 79            last_outs = outs
 80
 81        boxes, labels, probs = yolo_process_output(outs, yolov3_tiny_anchors, len(classes))
 82
 83        if last_boxes is not None:
 84            # Eliminate the boxes with low confidence and overlaped boxes
 85            if last_boxes.size > 0 and boxes.size > 0:
 86                indexes = cv2.dnn.NMSBoxes(np.vstack((boxes, last_boxes)).tolist(), np.hstack((np.array(probs), np.array(last_probs))), 0.5, 0.4)
 87                if len(indexes) > 0:
 88                    indexes = indexes.flatten()
 89
 90        last_boxes = np.copy(boxes)
 91        last_probs = np.copy(probs)
 92
 93        # Draw bounding boxes
 94        out_img = cv2.cvtColor(origin_cv_image, cv2.COLOR_BGR2RGB)
 95        out_img = out_img.astype(np.float32) / 255.0
 96        height, width, _ = out_img.shape
 97        noise = attack.noise
 98
 99        # Resize the noise to the same shape as the input image
100        # noise_r = bilinear_resize(noise[:, :, 0], height, width)
101        # noise_g = bilinear_resize(noise[:, :, 1], height, width)
102        # noise_b = bilinear_resize(noise[:, :, 2], height, width)
103        # noise = np.dstack((noise_r, noise_g, noise_b))
104
105        # out_img = out_img + noise
106        out_img = np.clip(out_img, 0, 1)
107
108        out_img = (out_img * 255.0).astype(np.uint8)
109
110        out_img = cv2.cvtColor(out_img, cv2.COLOR_RGB2BGR)
111        if len(boxes) > 0:
112            out_img = draw_bounding_boxes(out_img, boxes, labels, classes, probs);
113
114        cv2.namedWindow("result", cv2.WINDOW_NORMAL)
115        cv2.imshow("result", out_img)
116
117        if (cv2.waitKey(1) & 0xFF == ord('q')):
118            break