Direction aware convexity measure

depth_segmentation/CMakeLists.txt

@@ -73,16 +73,17 @@ target_link_libraries(${PROJECT_NAME}_node ${PROJECT_NAME})
 
 # COPY TEST DATA
 # TODO(ff): We should move the test data to an external repo or a cloud at some point.
-# add_custom_target(test_data)
-# add_custom_command(TARGET test_data
-#                    COMMAND rm -rf test_data
-#                    COMMAND mkdir -p test_data
-#                    COMMAND cp -r ${CMAKE_SOURCE_DIR}/../test_data/depth_segmentation/*
-#                    test_data/ || :)
+add_custom_target(test_data)
+add_custom_command(TARGET test_data
+                   COMMAND rm -rf test_data
+                   COMMAND mkdir -p test_data
+                   COMMAND cp -r ${CMAKE_SOURCE_DIR}/test/data/*
+                   test_data/ || :)
 
 # UNIT TESTS
 catkin_add_gtest(test_depth_segmentation test/test_depth_segmentation.cpp)
 target_link_libraries(test_depth_segmentation ${PROJECT_NAME} pthread)
+add_dependencies(test_depth_segmentation test_data)
 
 cs_install()
 cs_export()

depth_segmentation/cfg/DepthSegmenter.cfg

@@ -12,6 +12,12 @@ general_params.add("dilate_depth_image", bool_t, 0,
 general_params.add("dilation_size", int_t, 0,
                    "Size of the dilation for the registered depth image.", 1, 1,
                    15)
+general_params.add("visualize_segmented_scene", bool_t, 0,
+                   "Publish segmented scene.", False)
+general_params.add("display_rgb_image", bool_t, 0, "Visualize RGB image.",
+                   False)
+general_params.add("display_depth_image", bool_t, 0, "Visualize depth image.",
+                   False)
 
 # Surface normal estimation parameters.
 surface_normal = gen.add_group("surface_normal")
@@ -89,10 +95,21 @@ min_convexity.add("min_convexity_step_size", int_t, 0,
                   "The pixel step size of the min convexity filter.", 1, 1, 9)
 min_convexity.add("min_convexity_threshold", double_t, 0,
                   "The threshold of what is still convex.", 0.97, 0.0, 1.0)
+min_convexity.add(
+    "min_convexity_projection_threshold", double_t, 0,
+    "The threshold for dot product between normal and projection plane normal. If the value is close to 1.0 it means that the projected normal is less reliable and should be discarded.",
+    0.8, 0.0, 1.0)
 min_convexity.add("min_convexity_mask_threshold", double_t, 0,
                   "The mask threshold for convexity.", -0.0005, -0.1, 0.1)
 min_convexity.add("min_convexity_display", bool_t, 0,
                   "Display the min convexity map.", False)
+min_convexity.add(
+    "min_convexity_use_projected_normals_for_convexity_condition", bool_t, 0,
+    "If True uses projected normals to determine if point is convex.", False)
+min_convexity.add(
+    "min_convexity_use_projected_normals_for_convexity_measure", bool_t, 0,
+    "If True uses projected normals dot product as measure for convexity.",
+    False)
 
 # Final edge map parameters.
 final_edge = gen.add_group("final_edge")

depth_segmentation/cfg/asl_office_floor_config.yaml

@@ -9,6 +9,8 @@ depth_discontinuity_ratio: 0.01
 depth_discontinuity_use_depth_discontinuity: true
 dilate_depth_image: false
 dilation_size: 1
+display_rgb_image: false
+display_depth_image: false
 final_edge_display: false
 final_edge_morphological_closing_size: 1
 final_edge_morphological_opening_size: 1
@@ -37,6 +39,8 @@ min_convexity_step_size: 1
 min_convexity_threshold: 0.97
 min_convexity_use_min_convexity: true
 min_convexity_use_morphological_opening: true
+min_convexity_use_projected_normals_for_convexity_condition: false
+min_convexity_use_projected_normals_for_convexity_measure: false
 min_convexity_use_threshold: true
 min_convexity_window_size: 5
 normals_display: false

depth_segmentation/cfg/primesense_config.yaml

@@ -2,6 +2,10 @@ depth_discontinuity_display: true
 depth_discontinuity_kernel_size: 3
 depth_discontinuity_ratio: 0.05
 depth_discontinuity_use_depth_discontinuity: true
+dilate_depth_image: false
+dilation_size: 1
+display_rgb_image: false
+display_depth_image: false
 final_edge_display: false
 final_edge_morphological_closing_size: 1
 final_edge_morphological_opening_size: 1
@@ -25,12 +29,16 @@ max_distance_window_size: 1
 min_convexity_display: true
 min_convexity_mask_threshold: -0.0005
 min_convexity_morphological_opening_size: 1
+min_convexity_projection_threshold: 0.8
 min_convexity_step_size: 1
 min_convexity_threshold: 0.94
 min_convexity_use_morphological_opening: true
+min_convexity_use_projected_normals_for_convexity_condition: false
+min_convexity_use_projected_normals_for_convexity_measure: false
 min_convexity_use_threshold: true
 min_convexity_window_size: 5
 normals_display: true
 normals_distance_factor_threshold: 0.05
 normals_method: 3
 normals_window_size: 13
+visualize_segmented_scene: false

depth_segmentation/cfg/scenenet_config.yaml

@@ -7,6 +7,10 @@ depth_discontinuity_display: true
 depth_discontinuity_kernel_size: 3
 depth_discontinuity_ratio: 0.03
 depth_discontinuity_use_depth_discontinuity: true
+dilate_depth_image: false
+dilation_size: 1
+display_rgb_image: false
+display_depth_image: false
 final_edge_display: false
 final_edge_morphological_closing_size: 1
 final_edge_morphological_opening_size: 1
@@ -30,12 +34,16 @@ max_distance_window_size: 1
 min_convexity_display: true
 min_convexity_mask_threshold: -0.0005
 min_convexity_morphological_opening_size: 1
+min_convexity_projection_threshold: 0.8
 min_convexity_step_size: 1
 min_convexity_threshold: 0.94
 min_convexity_use_morphological_opening: true
+min_convexity_use_projected_normals_for_convexity_condition: false
+min_convexity_use_projected_normals_for_convexity_measure: false
 min_convexity_use_threshold: true
 min_convexity_window_size: 5
 normals_display: true
 normals_distance_factor_threshold: 0.05
 normals_method: 3
 normals_window_size: 13
+visualize_segmented_scene: false

depth_segmentation/cfg/scenenn_config.yaml

@@ -9,6 +9,8 @@ depth_discontinuity_ratio: 0.01
 depth_discontinuity_use_depth_discontinuity: true
 dilate_depth_image: false
 dilation_size: 1
+display_rgb_image: false
+display_depth_image: false
 final_edge_display: false
 final_edge_morphological_closing_size: 1
 final_edge_morphological_opening_size: 1
@@ -33,10 +35,13 @@ max_distance_window_size: 1
 min_convexity_display: false
 min_convexity_mask_threshold: -0.0005
 min_convexity_morphological_opening_size: 1
+min_convexity_projection_threshold: 0.8
 min_convexity_step_size: 1
 min_convexity_threshold: 0.97
 min_convexity_use_min_convexity: true
 min_convexity_use_morphological_opening: true
+min_convexity_use_projected_normals_for_convexity_condition: false
+min_convexity_use_projected_normals_for_convexity_measure: false
 min_convexity_use_threshold: true
 min_convexity_window_size: 5
 normals_display: false

depth_segmentation/include/depth_segmentation/common.h

@@ -83,7 +83,10 @@ struct MinConvexityMapParams {
   bool use_morphological_opening = true;
   bool use_threshold = true;
   double threshold = 0.97;
+  double projection_threshold = 0.8;
   double mask_threshold = -0.0005;
+  bool use_projected_normals_for_convexity_condition = false;
+  bool use_projected_normals_for_convexity_measure = false;
 };
 
 struct FinalEdgeMapParams {
@@ -137,6 +140,8 @@ struct Params {
   SurfaceNormalParams normals;
   SemanticInstanceSegmentationParams semantic_instance_segmentation;
   bool visualize_segmented_scene = false;
+  bool display_rgb_image = false;
+  bool display_depth_image = false;
 };
 
 void visualizeDepthMap(const cv::Mat& depth_map, cv::viz::Viz3d* viz_3d) {
@@ -163,7 +168,7 @@ void visualizeDepthMapWithNormals(const cv::Mat& depth_map,
   viz_3d->showWidget("cloud",
                      cv::viz::WCloud(depth_map, cv::viz::Color::red()));
   viz_3d->showWidget("normals",
-                     cv::viz::WCloudNormals(depth_map, normals, 50, 0.02f,
+                     cv::viz::WCloudNormals(depth_map, normals, 1, 0.02f,
                                             cv::viz::Color::green()));
   viz_3d->showWidget("coo", cv::viz::WCoordinateSystem(1.5));
   viz_3d->spinOnce(0, true);
@@ -196,6 +201,7 @@ void computeCovariance(const cv::Mat& neighborhood, const cv::Vec3f& mean,
   covariance->at<float>(1, 0) = covariance->at<float>(0, 1);
   covariance->at<float>(2, 0) = covariance->at<float>(0, 2);
   covariance->at<float>(2, 1) = covariance->at<float>(1, 2);
+  CHECK_GT(cv::countNonZero(*covariance), 0) << *covariance;
 }
 
 size_t findNeighborhood(const cv::Mat& depth_map, const size_t window_size,
@@ -306,6 +312,7 @@ void computeOwnNormals(const SurfaceNormalParams& params,
           normals->at<cv::Vec3f>(y, x)[coordinate] =
               eigenvectors.at<float>(n_th_eigenvector, coordinate);
         }
+        CHECK_NEAR(cv::norm(normals->at<cv::Vec3f>(y, x)), 1.0, 1e-5);
         // Re-Orient normals to point towards camera.
         if (normals->at<cv::Vec3f>(y, x)[2] > 0.0f) {
           normals->at<cv::Vec3f>(y, x) = -normals->at<cv::Vec3f>(y, x);

depth_segmentation/include/depth_segmentation/depth_segmentation.h

@@ -182,6 +182,7 @@ class DepthSegmenter {
   cv::rgbd::RgbdNormals rgbd_normals_;
   std::vector<cv::Scalar> colors_;
   std::vector<int> labels_;
+  cv::Mat display_image_;
 };
 
 // TODO(ntonci): Make a unit test.