Feature/face antispoofing fix

include/daisykit/common/types/face_extended.h

@@ -27,7 +27,7 @@ namespace types {
 /// This is used for face recognition.
 class FaceExtended : public Face {
  public:
-  float liveness_score;
+  float liveness_score = -1; 
 
   cv::Mat aligned_face;  /// Aligned face. For increasing recognition accuracy,
                          /// the face should be aligned before recognition.

include/daisykit/common/visualizers/face_visualizer.h

@@ -37,7 +37,25 @@ class FaceVisualizer {
           mask = "Mask";
         }
       }
-      BaseVisualizer::DrawRoundedBox(img, static_cast<types::Box>(face), mask);
+
+      std::cout << face.liveness_score << std::endl;
+      std::string liveness_detection;
+      cv::Scalar line_color = cv::Scalar(0, 255, 0);
+      if (face.liveness_score >= 0.0) {
+        if (face.liveness_score < 0.975f) {
+          liveness_detection =
+              "(Fake face): " + std::to_string(face.liveness_score);
+          line_color = cv::Scalar(0, 0, 255);
+        } else {
+          liveness_detection =
+              "(Real face): " + std::to_string(face.liveness_score);
+          line_color = cv::Scalar(0, 255, 0);
+        }
+      }
+      std::string text_to_show = mask + liveness_detection;
+      BaseVisualizer::DrawRoundedBox(img, static_cast<types::Box>(face),
+                                     text_to_show, line_color);
+
       if (with_landmark) {
         DrawLandmark(img, face.landmark);
       }

include/daisykit/flows/face_detector_flow.h

@@ -38,8 +38,8 @@ class FaceDetectorFlow {
                    bool show_fps = false);
 #endif
   ~FaceDetectorFlow();
-  std::vector<types::Face> Process(const cv::Mat& rgb);
-  void DrawResult(cv::Mat& rgb, std::vector<types::Face>& faces);
+  std::vector<types::FaceExtended> Process(const cv::Mat& rgb);
+  void DrawResult(cv::Mat& rgb, std::vector<types::FaceExtended>& faces);
 
  private:
   bool with_landmark_ = false;

include/daisykit/graphs/nodes/models/face_detector_node.h

@@ -46,13 +46,14 @@ class FaceDetectorNode : public Node {
     cv::Mat img = *in_packet->GetData<cv::Mat>();
 
     // Process
-    std::shared_ptr<std::vector<daisykit::types::Face>> result =
-        std::make_shared<std::vector<daisykit::types::Face>>();
+    std::shared_ptr<std::vector<daisykit::types::FaceExtended>> result =
+        std::make_shared<std::vector<daisykit::types::FaceExtended>>();
     face_detector_->Predict(img, *result);
 
     // Convert to output packet
     utils::TimePoint timestamp = daisykit::utils::Timer::Now();
-    out_packet = Packet::MakePacket<std::vector<daisykit::types::Face>>(result);
+    out_packet =
+        Packet::MakePacket<std::vector<daisykit::types::FaceExtended>>(result);
   }
 
   void Tick() {

include/daisykit/graphs/nodes/models/face_landmark_detector_node.h

@@ -52,8 +52,9 @@ class FacialLandmarkDetectorNode : public Node {
     PrepareInputs(inputs);
 
     // Get faces result
-    std::shared_ptr<std::vector<daisykit::types::Face>> faces;
-    faces = inputs["faces"]->GetData<std::vector<daisykit::types::Face>>();
+    std::shared_ptr<std::vector<daisykit::types::FaceExtended>> faces;
+    faces =
+        inputs["faces"]->GetData<std::vector<daisykit::types::FaceExtended>>();
 
     // Get image
     cv::Mat img = *inputs["image"]->GetData<cv::Mat>();

include/daisykit/models/face_detector.h

@@ -53,7 +53,8 @@ class FaceDetector : public NCNNModel, public ImageModel {
 
   /// Detect faces in an image.
   /// Return 0 on success, otherwise return error code.
-  int Predict(const cv::Mat& image, std::vector<daisykit::types::Face>& faces);
+  int Predict(const cv::Mat& image,
+              std::vector<daisykit::types::FaceExtended>& faces);
 
  private:
   /// Preprocess image data to obtain net input.

include/daisykit/models/face_liveness_detector.h

@@ -45,11 +45,13 @@ class FaceLivenessDetector : public NCNNModel, public ImageModel {
                        const std::string& weight_file);
 #endif
 
-  int Predict(const cv::Mat& image, types::FaceExtended& faces);
+  int Predict(const cv::Mat& image,
+              std::vector<daisykit::types::FaceExtended>& faces);
 
  private:
   void Preprocess(const cv::Mat& image, ncnn::Mat& net_input);
-  cv::Rect CalculateBox(daisykit::types::FaceExtended& face_box, int w, int h);
+  std::vector<cv::Rect> CalculateBox(
+      std::vector<daisykit::types::FaceExtended>& face_box, int w, int h);
 };
 }  // namespace models
 }  // namespace daisykit

include/daisykit/models/facial_landmark_detector.h

@@ -59,7 +59,7 @@ class FacialLandmarkDetector : public NCNNModel, public ImageModel {
   /// Modify faces vector to add landmark info. Return 0 on success, otherwise
   /// return the number of inference errors.
   int PredictMulti(const cv::Mat& image,
-                   std::vector<daisykit::types::Face>& faces);
+                   std::vector<daisykit::types::FaceExtended>& faces);
 
  private:
   /// Preprocess image data to obtain net input.

src/examples/demo_face_detector.cpp

@@ -44,7 +44,7 @@ int main(int, char**) {
     cv::Mat rgb;
     cv::cvtColor(frame, rgb, cv::COLOR_BGR2RGB);
 
-    std::vector<types::Face> faces = flow.Process(rgb);
+    std::vector<types::FaceExtended> faces = flow.Process(rgb);
     flow.DrawResult(rgb, faces);
 
     cv::Mat draw;

src/examples/demo_face_detector_scrfd.cpp

@@ -30,21 +30,22 @@ using json = nlohmann::json;
 using namespace daisykit;
 using namespace daisykit::models;
 
-FaceDetectorSCRFD<types::Face>* face_detector =
-    new FaceDetectorSCRFD<types::Face>(
-        "models/face_detection/scrfd/scrfd_2.5g_1.param",
-        "models/face_detection/scrfd/scrfd_2.5g_1.bin", 640, 0.7, 0.5, false);
+FaceDetectorSCRFD<types::FaceExtended>* face_detector =
+    new FaceDetectorSCRFD<types::FaceExtended>(
+        "models/face_detection_scrfd/scrfd_2.5g_1.param",
+        "models/face_detection_scrfd/scrfd_2.5g_1.bin", 640, 0.7, 0.5, false);
 
 int main(int, char**) {
   Mat frame;
   VideoCapture cap(0);
 
-  std::vector<types::Face> faces;
+  std::vector<types::FaceExtended> faces;
   while (1) {
     cap >> frame;
     face_detector->Predict(frame, faces);
     cv::Mat draw = frame.clone();
-    visualizers::FaceVisualizer<types::Face>::DrawFace(draw, faces, true);
+    visualizers::FaceVisualizer<types::FaceExtended>::DrawFace(draw, faces,
+                                                               true);
     imshow("Image", draw);
     waitKey(1);
   }

src/examples/demo_face_liveness_detection.cpp

@@ -29,7 +29,7 @@ using namespace std;
 using namespace daisykit;
 using namespace daisykit::models;
 
-FaceLivenessDetector* face_liveness_detector = new FaceLivenessDetector(
+FaceLivenessDetector* face_liveness_detector_2 = new FaceLivenessDetector(
     "models/face_antispoofing/minivision/model_2.param",
     "models/face_antispoofing/minivision/model_2.bin", 80, 80, true);
 
@@ -44,24 +44,13 @@ int main(int, char**) {
 
   std::vector<types::FaceExtended> faces;
   while (1) {
-    int thickness = 2;
     cap >> frame;
     face_detector->Predict(frame, faces);
-    int face_count = 0;
-    for (auto face : faces) {
-      face.liveness_score = 0;
-      face_liveness_detector->Predict(frame, face);
-
-      Point tl(face.x, face.y);
-      Point br(face.x + face.w, face.y + face.h);
-      if (face.liveness_score < 0.97) {
-        rectangle(frame, tl, br, Scalar(0, 0, 255), thickness, LINE_8);
-      } else {
-        rectangle(frame, tl, br, Scalar(0, 255, 0), thickness, LINE_8);
-      }
-    }
-
-    imshow("Image", frame);
+    face_liveness_detector_2->Predict(frame, faces);
+    cv::Mat draw = frame.clone();
+    visualizers::FaceVisualizer<types::FaceExtended>::DrawFace(draw, faces,
+                                                               false);
+    imshow("Image", draw);
     waitKey(1);
   }
 

src/flows/face_detector_flow.cpp

@@ -75,9 +75,9 @@ FaceDetectorFlow::~FaceDetectorFlow() {
   facial_landmark_detector_ = nullptr;
 }
 
-std::vector<types::Face> FaceDetectorFlow::Process(const cv::Mat& rgb) {
+std::vector<types::FaceExtended> FaceDetectorFlow::Process(const cv::Mat& rgb) {
   // Detect faces
-  std::vector<types::Face> faces;
+  std::vector<types::FaceExtended> faces;
   face_detector_->Predict(rgb, faces);
 
   // Detect landmarks
@@ -92,9 +92,9 @@ std::vector<types::Face> FaceDetectorFlow::Process(const cv::Mat& rgb) {
 }
 
 void FaceDetectorFlow::DrawResult(cv::Mat& rgb,
-                                  std::vector<types::Face>& faces) {
+                                  std::vector<types::FaceExtended>& faces) {
   // Draw face bounding boxes and keypoints
-  visualizers::FaceVisualizer<types::Face>::DrawFace(rgb, faces, true);
+  visualizers::FaceVisualizer<types::FaceExtended>::DrawFace(rgb, faces, true);
   if (show_fps_)
     visualizers::BaseVisualizer::PutText(
         rgb, std::string("FPS: ") + std::to_string(profiler.CurrentFPS()),

src/models/face_detector.cpp

@@ -91,7 +91,7 @@ void FaceDetector::Preprocess(const cv::Mat& image, ncnn::Mat& net_input) {
 }
 
 int FaceDetector::Predict(const cv::Mat& image,
-                          std::vector<daisykit::types::Face>& faces) {
+                          std::vector<daisykit::types::FaceExtended>& faces) {
   // Preprocess
   ncnn::Mat in;
   Preprocess(image, in);

src/models/face_liveness_detector.cpp

@@ -39,52 +39,56 @@ FaceLivenessDetector::FaceLivenessDetector(const std::string& param_file,
       ImageModel(input_width, input_height) {}
 
 #if __ANDROID__
-LivenessDetector::LivenessDetector(AAssetManager* mgr,
-                                   const std::string& param_file,
-                                   const std::string& weight_file)
+FaceLivenessDetector::FaceLivenessDetector(AAssetManager* mgr,
+                                           const std::string& param_file,
+                                           const std::string& weight_file)
     : NCNNModel(param_buffer, weight_buffer, use_gpu),
       ImageModel(input_width, input_height) {}
 #endif
 
-cv::Rect FaceLivenessDetector::CalculateBox(types::FaceExtended& face_box,
-                                            int w, int h) {
-  float scale_ = 4.0;
-  float scale = std::min(scale_, std::min((w - 1) / (float)face_box.w,
-                                          (h - 1) / (float)face_box.h));
-  int box_center_x = face_box.w / 2 + face_box.x;
-  int box_center_y = face_box.h / 2 + face_box.y;
-
-  int new_width = static_cast<int>(face_box.w * scale);
-  int new_height = static_cast<int>(face_box.h * scale);
-
-  int left_top_x = box_center_x - new_width / 2;
-  int left_top_y = box_center_y - new_height / 2;
-  int right_bottom_x = box_center_x + new_width / 2;
-  int right_bottom_y = box_center_y + new_height / 2;
-
-  if (left_top_x < 0) {
-    right_bottom_x -= left_top_x;
-    left_top_x = 0;
+std::vector<cv::Rect> FaceLivenessDetector::CalculateBox(
+    std::vector<types::FaceExtended>& face_boxes, int w, int h) {
+  std::vector<cv::Rect> face_boxes_out;
+  for (auto face_box : face_boxes) {
+    float scale_ = 4.0;
+    float scale = std::min(scale_, std::min((w - 1) / (float)face_box.w,
+                                            (h - 1) / (float)face_box.h));
+    int box_center_x = face_box.w / 2 + face_box.x;
+    int box_center_y = face_box.h / 2 + face_box.y;
+
+    int new_width = static_cast<int>(face_box.w * scale);
+    int new_height = static_cast<int>(face_box.h * scale);
+
+    int left_top_x = box_center_x - new_width / 2;
+    int left_top_y = box_center_y - new_height / 2;
+    int right_bottom_x = box_center_x + new_width / 2;
+    int right_bottom_y = box_center_y + new_height / 2;
+
+    if (left_top_x < 0) {
+      right_bottom_x -= left_top_x;
+      left_top_x = 0;
+    }
+
+    if (left_top_y < 0) {
+      right_bottom_y -= left_top_y;
+      left_top_y = 0;
+    }
+
+    if (right_bottom_x >= w) {
+      int s = right_bottom_x - w + 1;
+      left_top_x -= s;
+      right_bottom_x -= s;
+    }
+
+    if (right_bottom_y >= h) {
+      int s = right_bottom_y - h + 1;
+      left_top_y -= s;
+      right_bottom_y -= s;
+    }
+    face_boxes_out.emplace_back(
+        cv::Rect(left_top_x, left_top_y, new_width, new_height));
   }
-
-  if (left_top_y < 0) {
-    right_bottom_y -= left_top_y;
-    left_top_y = 0;
-  }
-
-  if (right_bottom_x >= w) {
-    int s = right_bottom_x - w + 1;
-    left_top_x -= s;
-    right_bottom_x -= s;
-  }
-
-  if (right_bottom_y >= h) {
-    int s = right_bottom_y - h + 1;
-    left_top_y -= s;
-    right_bottom_y -= s;
-  }
-
-  return cv::Rect(left_top_x, left_top_y, new_width, new_height);
+  return face_boxes_out;
 }
 
 void FaceLivenessDetector::Preprocess(const cv::Mat& image,
@@ -94,24 +98,24 @@ void FaceLivenessDetector::Preprocess(const cv::Mat& image,
 }
 
 int FaceLivenessDetector::Predict(const cv::Mat& image,
-                                  types::FaceExtended& faces) {
-  float liveness_score = 0.f;
-  cv::Mat roi;
-  cv::Rect rect = CalculateBox(faces, image.cols, image.rows);
-  cv::resize(image(rect), roi, cv::Size(80, 80));
-
-  // Preprocess
-  ncnn::Mat in;
-  Preprocess(roi, in);
-
-  ncnn::Mat out;
-  int result = Infer(in, out, "data", "softmax");
-  if (result != 0) return 0;
-
-  // Post process
-  liveness_score += out.row(0)[1];
-  faces.liveness_score = liveness_score;
-  // Model Inference
+                                  std::vector<types::FaceExtended>& faces) {
+  std::vector<cv::Rect> rects = CalculateBox(faces, image.cols, image.rows);
+  for (int i = 0; i < rects.size(); i++) {
+    cv::Mat roi;
+    cv::resize(image(rects[i]), roi, cv::Size(80, 80));
+
+    // Preprocess
+    ncnn::Mat in;
+    Preprocess(roi, in);
+
+    // Model Inference
+    ncnn::Mat out;
+    int result = Infer(in, out, "data", "softmax");
+    if (result != 0) return 0;
+
+    // Post process
+    faces[i].liveness_score = out.row(0)[1];
+  }
 
   return 0;
 }

src/models/facial_landmark_detector.cpp

@@ -84,8 +84,8 @@ int FacialLandmarkDetector::Predict(const cv::Mat& image,
   return 0;
 }
 
-int FacialLandmarkDetector::PredictMulti(const cv::Mat& image,
-                                         std::vector<types::Face>& faces) {
+int FacialLandmarkDetector::PredictMulti(
+    const cv::Mat& image, std::vector<types::FaceExtended>& faces) {
   int num_errors = 0;
   int img_width = image.cols;
   int img_height = image.rows;