diff --git a/include/daisykit/common/types/face_extended.h b/include/daisykit/common/types/face_extended.h index 8eff931f..f8d0af5f 100644 --- a/include/daisykit/common/types/face_extended.h +++ b/include/daisykit/common/types/face_extended.h @@ -27,7 +27,7 @@ namespace types { /// This is used for face recognition. class FaceExtended : public Face { public: - float liveness_score; + float liveness_score = -1; cv::Mat aligned_face; /// Aligned face. For increasing recognition accuracy, /// the face should be aligned before recognition. diff --git a/include/daisykit/common/visualizers/face_visualizer.h b/include/daisykit/common/visualizers/face_visualizer.h index 0fc8ae02..0c178539 100644 --- a/include/daisykit/common/visualizers/face_visualizer.h +++ b/include/daisykit/common/visualizers/face_visualizer.h @@ -37,7 +37,25 @@ class FaceVisualizer { mask = "Mask"; } } - BaseVisualizer::DrawRoundedBox(img, static_cast(face), mask); + + std::cout << face.liveness_score << std::endl; + std::string liveness_detection; + cv::Scalar line_color = cv::Scalar(0, 255, 0); + if (face.liveness_score >= 0.0) { + if (face.liveness_score < 0.975f) { + liveness_detection = + "(Fake face): " + std::to_string(face.liveness_score); + line_color = cv::Scalar(0, 0, 255); + } else { + liveness_detection = + "(Real face): " + std::to_string(face.liveness_score); + line_color = cv::Scalar(0, 255, 0); + } + } + std::string text_to_show = mask + liveness_detection; + BaseVisualizer::DrawRoundedBox(img, static_cast(face), + text_to_show, line_color); + if (with_landmark) { DrawLandmark(img, face.landmark); } diff --git a/include/daisykit/flows/face_detector_flow.h b/include/daisykit/flows/face_detector_flow.h index cfb0d25e..ef7c2547 100644 --- a/include/daisykit/flows/face_detector_flow.h +++ b/include/daisykit/flows/face_detector_flow.h @@ -38,8 +38,8 @@ class FaceDetectorFlow { bool show_fps = false); #endif ~FaceDetectorFlow(); - std::vector Process(const cv::Mat& rgb); - void DrawResult(cv::Mat& rgb, std::vector& faces); + std::vector Process(const cv::Mat& rgb); + void DrawResult(cv::Mat& rgb, std::vector& faces); private: bool with_landmark_ = false; diff --git a/include/daisykit/graphs/nodes/models/face_detector_node.h b/include/daisykit/graphs/nodes/models/face_detector_node.h index bac2b6d1..b019319e 100644 --- a/include/daisykit/graphs/nodes/models/face_detector_node.h +++ b/include/daisykit/graphs/nodes/models/face_detector_node.h @@ -46,13 +46,14 @@ class FaceDetectorNode : public Node { cv::Mat img = *in_packet->GetData(); // Process - std::shared_ptr> result = - std::make_shared>(); + std::shared_ptr> result = + std::make_shared>(); face_detector_->Predict(img, *result); // Convert to output packet utils::TimePoint timestamp = daisykit::utils::Timer::Now(); - out_packet = Packet::MakePacket>(result); + out_packet = + Packet::MakePacket>(result); } void Tick() { diff --git a/include/daisykit/graphs/nodes/models/face_landmark_detector_node.h b/include/daisykit/graphs/nodes/models/face_landmark_detector_node.h index c129519c..2f704b1c 100644 --- a/include/daisykit/graphs/nodes/models/face_landmark_detector_node.h +++ b/include/daisykit/graphs/nodes/models/face_landmark_detector_node.h @@ -52,8 +52,9 @@ class FacialLandmarkDetectorNode : public Node { PrepareInputs(inputs); // Get faces result - std::shared_ptr> faces; - faces = inputs["faces"]->GetData>(); + std::shared_ptr> faces; + faces = + inputs["faces"]->GetData>(); // Get image cv::Mat img = *inputs["image"]->GetData(); diff --git a/include/daisykit/models/face_detector.h b/include/daisykit/models/face_detector.h index c43cd45c..0ba9e176 100644 --- a/include/daisykit/models/face_detector.h +++ b/include/daisykit/models/face_detector.h @@ -53,7 +53,8 @@ class FaceDetector : public NCNNModel, public ImageModel { /// Detect faces in an image. /// Return 0 on success, otherwise return error code. - int Predict(const cv::Mat& image, std::vector& faces); + int Predict(const cv::Mat& image, + std::vector& faces); private: /// Preprocess image data to obtain net input. diff --git a/include/daisykit/models/face_liveness_detector.h b/include/daisykit/models/face_liveness_detector.h index d9da34e9..73b53043 100644 --- a/include/daisykit/models/face_liveness_detector.h +++ b/include/daisykit/models/face_liveness_detector.h @@ -45,11 +45,13 @@ class FaceLivenessDetector : public NCNNModel, public ImageModel { const std::string& weight_file); #endif - int Predict(const cv::Mat& image, types::FaceExtended& faces); + int Predict(const cv::Mat& image, + std::vector& faces); private: void Preprocess(const cv::Mat& image, ncnn::Mat& net_input); - cv::Rect CalculateBox(daisykit::types::FaceExtended& face_box, int w, int h); + std::vector CalculateBox( + std::vector& face_box, int w, int h); }; } // namespace models } // namespace daisykit diff --git a/include/daisykit/models/facial_landmark_detector.h b/include/daisykit/models/facial_landmark_detector.h index b2f70be9..b27fcee1 100644 --- a/include/daisykit/models/facial_landmark_detector.h +++ b/include/daisykit/models/facial_landmark_detector.h @@ -59,7 +59,7 @@ class FacialLandmarkDetector : public NCNNModel, public ImageModel { /// Modify faces vector to add landmark info. Return 0 on success, otherwise /// return the number of inference errors. int PredictMulti(const cv::Mat& image, - std::vector& faces); + std::vector& faces); private: /// Preprocess image data to obtain net input. diff --git a/src/examples/demo_face_detector.cpp b/src/examples/demo_face_detector.cpp index 8ea91eda..f6cdbdf8 100644 --- a/src/examples/demo_face_detector.cpp +++ b/src/examples/demo_face_detector.cpp @@ -44,7 +44,7 @@ int main(int, char**) { cv::Mat rgb; cv::cvtColor(frame, rgb, cv::COLOR_BGR2RGB); - std::vector faces = flow.Process(rgb); + std::vector faces = flow.Process(rgb); flow.DrawResult(rgb, faces); cv::Mat draw; diff --git a/src/examples/demo_face_detector_scrfd.cpp b/src/examples/demo_face_detector_scrfd.cpp index 4cae4673..34277258 100644 --- a/src/examples/demo_face_detector_scrfd.cpp +++ b/src/examples/demo_face_detector_scrfd.cpp @@ -30,21 +30,22 @@ using json = nlohmann::json; using namespace daisykit; using namespace daisykit::models; -FaceDetectorSCRFD* face_detector = - new FaceDetectorSCRFD( - "models/face_detection/scrfd/scrfd_2.5g_1.param", - "models/face_detection/scrfd/scrfd_2.5g_1.bin", 640, 0.7, 0.5, false); +FaceDetectorSCRFD* face_detector = + new FaceDetectorSCRFD( + "models/face_detection_scrfd/scrfd_2.5g_1.param", + "models/face_detection_scrfd/scrfd_2.5g_1.bin", 640, 0.7, 0.5, false); int main(int, char**) { Mat frame; VideoCapture cap(0); - std::vector faces; + std::vector faces; while (1) { cap >> frame; face_detector->Predict(frame, faces); cv::Mat draw = frame.clone(); - visualizers::FaceVisualizer::DrawFace(draw, faces, true); + visualizers::FaceVisualizer::DrawFace(draw, faces, + true); imshow("Image", draw); waitKey(1); } diff --git a/src/examples/demo_face_liveness_detection.cpp b/src/examples/demo_face_liveness_detection.cpp index 02de1a36..9b45b614 100644 --- a/src/examples/demo_face_liveness_detection.cpp +++ b/src/examples/demo_face_liveness_detection.cpp @@ -29,7 +29,7 @@ using namespace std; using namespace daisykit; using namespace daisykit::models; -FaceLivenessDetector* face_liveness_detector = new FaceLivenessDetector( +FaceLivenessDetector* face_liveness_detector_2 = new FaceLivenessDetector( "models/face_antispoofing/minivision/model_2.param", "models/face_antispoofing/minivision/model_2.bin", 80, 80, true); @@ -44,24 +44,13 @@ int main(int, char**) { std::vector faces; while (1) { - int thickness = 2; cap >> frame; face_detector->Predict(frame, faces); - int face_count = 0; - for (auto face : faces) { - face.liveness_score = 0; - face_liveness_detector->Predict(frame, face); - - Point tl(face.x, face.y); - Point br(face.x + face.w, face.y + face.h); - if (face.liveness_score < 0.97) { - rectangle(frame, tl, br, Scalar(0, 0, 255), thickness, LINE_8); - } else { - rectangle(frame, tl, br, Scalar(0, 255, 0), thickness, LINE_8); - } - } - - imshow("Image", frame); + face_liveness_detector_2->Predict(frame, faces); + cv::Mat draw = frame.clone(); + visualizers::FaceVisualizer::DrawFace(draw, faces, + false); + imshow("Image", draw); waitKey(1); } diff --git a/src/flows/face_detector_flow.cpp b/src/flows/face_detector_flow.cpp index 541eaa8e..30368503 100644 --- a/src/flows/face_detector_flow.cpp +++ b/src/flows/face_detector_flow.cpp @@ -75,9 +75,9 @@ FaceDetectorFlow::~FaceDetectorFlow() { facial_landmark_detector_ = nullptr; } -std::vector FaceDetectorFlow::Process(const cv::Mat& rgb) { +std::vector FaceDetectorFlow::Process(const cv::Mat& rgb) { // Detect faces - std::vector faces; + std::vector faces; face_detector_->Predict(rgb, faces); // Detect landmarks @@ -92,9 +92,9 @@ std::vector FaceDetectorFlow::Process(const cv::Mat& rgb) { } void FaceDetectorFlow::DrawResult(cv::Mat& rgb, - std::vector& faces) { + std::vector& faces) { // Draw face bounding boxes and keypoints - visualizers::FaceVisualizer::DrawFace(rgb, faces, true); + visualizers::FaceVisualizer::DrawFace(rgb, faces, true); if (show_fps_) visualizers::BaseVisualizer::PutText( rgb, std::string("FPS: ") + std::to_string(profiler.CurrentFPS()), diff --git a/src/models/face_detector.cpp b/src/models/face_detector.cpp index 6ba33370..47b2e883 100644 --- a/src/models/face_detector.cpp +++ b/src/models/face_detector.cpp @@ -91,7 +91,7 @@ void FaceDetector::Preprocess(const cv::Mat& image, ncnn::Mat& net_input) { } int FaceDetector::Predict(const cv::Mat& image, - std::vector& faces) { + std::vector& faces) { // Preprocess ncnn::Mat in; Preprocess(image, in); diff --git a/src/models/face_liveness_detector.cpp b/src/models/face_liveness_detector.cpp index c649a40e..2b5c2f7e 100644 --- a/src/models/face_liveness_detector.cpp +++ b/src/models/face_liveness_detector.cpp @@ -39,52 +39,56 @@ FaceLivenessDetector::FaceLivenessDetector(const std::string& param_file, ImageModel(input_width, input_height) {} #if __ANDROID__ -LivenessDetector::LivenessDetector(AAssetManager* mgr, - const std::string& param_file, - const std::string& weight_file) +FaceLivenessDetector::FaceLivenessDetector(AAssetManager* mgr, + const std::string& param_file, + const std::string& weight_file) : NCNNModel(param_buffer, weight_buffer, use_gpu), ImageModel(input_width, input_height) {} #endif -cv::Rect FaceLivenessDetector::CalculateBox(types::FaceExtended& face_box, - int w, int h) { - float scale_ = 4.0; - float scale = std::min(scale_, std::min((w - 1) / (float)face_box.w, - (h - 1) / (float)face_box.h)); - int box_center_x = face_box.w / 2 + face_box.x; - int box_center_y = face_box.h / 2 + face_box.y; - - int new_width = static_cast(face_box.w * scale); - int new_height = static_cast(face_box.h * scale); - - int left_top_x = box_center_x - new_width / 2; - int left_top_y = box_center_y - new_height / 2; - int right_bottom_x = box_center_x + new_width / 2; - int right_bottom_y = box_center_y + new_height / 2; - - if (left_top_x < 0) { - right_bottom_x -= left_top_x; - left_top_x = 0; +std::vector FaceLivenessDetector::CalculateBox( + std::vector& face_boxes, int w, int h) { + std::vector face_boxes_out; + for (auto face_box : face_boxes) { + float scale_ = 4.0; + float scale = std::min(scale_, std::min((w - 1) / (float)face_box.w, + (h - 1) / (float)face_box.h)); + int box_center_x = face_box.w / 2 + face_box.x; + int box_center_y = face_box.h / 2 + face_box.y; + + int new_width = static_cast(face_box.w * scale); + int new_height = static_cast(face_box.h * scale); + + int left_top_x = box_center_x - new_width / 2; + int left_top_y = box_center_y - new_height / 2; + int right_bottom_x = box_center_x + new_width / 2; + int right_bottom_y = box_center_y + new_height / 2; + + if (left_top_x < 0) { + right_bottom_x -= left_top_x; + left_top_x = 0; + } + + if (left_top_y < 0) { + right_bottom_y -= left_top_y; + left_top_y = 0; + } + + if (right_bottom_x >= w) { + int s = right_bottom_x - w + 1; + left_top_x -= s; + right_bottom_x -= s; + } + + if (right_bottom_y >= h) { + int s = right_bottom_y - h + 1; + left_top_y -= s; + right_bottom_y -= s; + } + face_boxes_out.emplace_back( + cv::Rect(left_top_x, left_top_y, new_width, new_height)); } - - if (left_top_y < 0) { - right_bottom_y -= left_top_y; - left_top_y = 0; - } - - if (right_bottom_x >= w) { - int s = right_bottom_x - w + 1; - left_top_x -= s; - right_bottom_x -= s; - } - - if (right_bottom_y >= h) { - int s = right_bottom_y - h + 1; - left_top_y -= s; - right_bottom_y -= s; - } - - return cv::Rect(left_top_x, left_top_y, new_width, new_height); + return face_boxes_out; } void FaceLivenessDetector::Preprocess(const cv::Mat& image, @@ -94,24 +98,24 @@ void FaceLivenessDetector::Preprocess(const cv::Mat& image, } int FaceLivenessDetector::Predict(const cv::Mat& image, - types::FaceExtended& faces) { - float liveness_score = 0.f; - cv::Mat roi; - cv::Rect rect = CalculateBox(faces, image.cols, image.rows); - cv::resize(image(rect), roi, cv::Size(80, 80)); - - // Preprocess - ncnn::Mat in; - Preprocess(roi, in); - - ncnn::Mat out; - int result = Infer(in, out, "data", "softmax"); - if (result != 0) return 0; - - // Post process - liveness_score += out.row(0)[1]; - faces.liveness_score = liveness_score; - // Model Inference + std::vector& faces) { + std::vector rects = CalculateBox(faces, image.cols, image.rows); + for (int i = 0; i < rects.size(); i++) { + cv::Mat roi; + cv::resize(image(rects[i]), roi, cv::Size(80, 80)); + + // Preprocess + ncnn::Mat in; + Preprocess(roi, in); + + // Model Inference + ncnn::Mat out; + int result = Infer(in, out, "data", "softmax"); + if (result != 0) return 0; + + // Post process + faces[i].liveness_score = out.row(0)[1]; + } return 0; } diff --git a/src/models/facial_landmark_detector.cpp b/src/models/facial_landmark_detector.cpp index 5068a4bb..59f434ca 100644 --- a/src/models/facial_landmark_detector.cpp +++ b/src/models/facial_landmark_detector.cpp @@ -84,8 +84,8 @@ int FacialLandmarkDetector::Predict(const cv::Mat& image, return 0; } -int FacialLandmarkDetector::PredictMulti(const cv::Mat& image, - std::vector& faces) { +int FacialLandmarkDetector::PredictMulti( + const cv::Mat& image, std::vector& faces) { int num_errors = 0; int img_width = image.cols; int img_height = image.rows;