diff --git a/mask_rcnn/.gitignore b/mask_rcnn/.gitignore new file mode 100644 index 00000000..fbb17b59 --- /dev/null +++ b/mask_rcnn/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +.vscode/ +.idea/ +*.h5 +*.pb +**/dataset/train/ +**/dataset/val/ +**/mask_rcnn/logs/ diff --git a/mask_rcnn/README.md b/mask_rcnn/README.md new file mode 100644 index 00000000..68e87b0b --- /dev/null +++ b/mask_rcnn/README.md @@ -0,0 +1,73 @@ +# Pointless Packaging W/ Mask R-CNN + +MASK R-CNN: https://github.com/matterport/Mask_RCNN + +``` +DIRECTORY STRUCTURE +. +|____.gitignore +|____dataset +| |____train - contains all training images (in Google Drive) +| |____TRAIN-MINI.json - VIA annotations for training images +| |____val - contains all vaidation images (in Google Drive) +| |____VAL-MINI.json - VIA annotations for validation images +| |____val_img_results - contains results of validation images after inference +| |____via.html - VIA annotator program +|____eval_on_val_set.py - performs inference on validation test, resuls in + `val_img_results` +|____logs - training logs from the mask r-cnn library +|____models - contains trained models +| |____mask_rcnn_final.h5 (256MB) - DOWNLOAD LINK BELOW +| |____.gitkeep - Dummy file. Just ignore it. +|____mrcnn - matterport/Mask_RCNN library +|____README.md +|____requirements.txt +|____test_images - images that can be tested by running `score.py` +|____trainer.py - train a dataset using the Mask R-CNN library +|____trainer_voc.py - train a dataset in PASCAL-VOC format using the Mask R-CNN library +| +|____score.py - RUN THIS script to obtain scores of images + containing pointless packaging. +``` + +- Download the latest trained model and place it in the `models/` directory. + - ### CLICK HERE TO DOWNLOAD TRAINED MODEL + - The model was trained for oly 50 epochs on 150 training images and 32 test images. + + +## score.py +``` +usage: score.py [-h] -m MODEL_SRC [-i IMG_SRC | -d DIR_SRC] [-v] + +Simple script that takes a trained MASK R-CNN model (.h5), pointless +packaging image/images and then generates score of the packaging purely based on +the area of the package relative to the item inside; using the provided model. + +optional arguments: + -h, --help show this help message and exit + -m MODEL_SRC, --model MODEL_SRC + Absolute/Relative path to the MASK R-CNN Model + -i IMG_SRC, --img IMG_SRC + Absolute/Relative path of the image. Cannot include + --dir argument. + -d DIR_SRC, --dir DIR_SRC + Absolute/Relative path of the directory containing the + images. Cannot include --img argument. + -v, --visualize Visualize the image. + +``` +### Example: +- Get score of a single image WITH visulatization. + - `python3 score.py -v -m models/mask_rcnn_final.h5 -i test_images/IMG_0.jpg` +- Get score for every single image in a directory WITH visualization. + - `python3 score.py -v -m models/mask_rcnn_final.h5 -d test_images/` +- Get score for every single image in a directory WITHOUT visualization. + - `python3 score.py -m models/mask_rcnn_final.h5 -d test_images/` + +## TODO: +- Need to come up with a proper scoring function. + Currently, `score.py` gives only the area of the + box and items in pixels. + +## RESULTS: +![Results](dataset/val_img_results/Figure_1.png) \ No newline at end of file diff --git a/mask_rcnn/dataset/TRAIN-MINI.json b/mask_rcnn/dataset/TRAIN-MINI.json new file mode 100644 index 00000000..9df8f179 --- /dev/null +++ b/mask_rcnn/dataset/TRAIN-MINI.json @@ -0,0 +1 @@ +{"_via_settings":{"ui":{"annotation_editor_height":25,"annotation_editor_fontsize":0.8,"leftsidebar_width":18,"image_grid":{"img_height":80,"rshape_fill":"none","rshape_fill_opacity":0.3,"rshape_stroke":"yellow","rshape_stroke_width":2,"show_region_shape":true,"show_image_policy":"all"},"image":{"region_label":"name","region_color":"__via_default_region_color__","region_label_font":"10px Sans","on_image_annotation_editor_placement":"NEAR_REGION"}},"core":{"buffer_size":"18","filepath":{},"default_filepath":"train/"},"project":{"name":"TRAIN"}},"_via_img_metadata":{"IMG_0.jpg52157":{"filename":"IMG_0.jpg","size":52157,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[263,278,266,251],"all_points_y":[137,139,196,192]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[197,414,414,192],"all_points_y":[92,86,248,238]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[134,194,199,415,414,497,501,411,412,188,192,129],"all_points_y":[94,92,24,13,86,86,247,246,298,298,232,227]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_1.jpg13491":{"filename":"IMG_1.jpg","size":13491,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[70,87,86,70],"all_points_y":[145,145,172,171]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[41,114,122,41],"all_points_y":[98,96,206,211]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[43,110,113,142,143,121,121,41,40,2,7,40],"all_points_y":[65,63,94,97,207,207,251,254,210,212,98,98]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_2.jpg14457":{"filename":"IMG_2.jpg","size":14457,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[50,73,87,64],"all_points_y":[141,118,134,156]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[39,110,112,44],"all_points_y":[81,83,189,196]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[38,106,109,136,138,112,110,45,44,5,2,38],"all_points_y":[43,48,83,86,183,187,220,233,196,199,79,80]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_3.jpg28329":{"filename":"IMG_3.jpg","size":28329,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[100,144,120,73],"all_points_y":[109,130,195,174]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[86,184,140,17],"all_points_y":[46,92,253,223]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[101,222,183,224,197,140,168,8,17,1,78,85],"all_points_y":[5,67,90,110,267,251,282,250,224,231,1,44]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_4.jpg18069":{"filename":"IMG_4.jpg","size":18069,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[109,154,150,107],"all_points_y":[101,106,151,150]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[61,168,168,68],"all_points_y":[72,66,221,225]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[58,165,168,216,214,169,172,67,67,16,4,61],"all_points_y":[15,8,67,56,241,221,281,279,223,228,74,72]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_5.jpg30039":{"filename":"IMG_5.jpg","size":30039,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[106,120,91,75],"all_points_y":[128,134,202,197]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[69,63,127,133],"all_points_y":[128,146,166,151]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[90,188,114,3],"all_points_y":[70,186,265,129]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[97,99,91,128,225,189,223,141,117,115,2,1,1,98],"all_points_y":[36,33,70,43,152,186,185,287,267,285,152,131,86,33]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_6.jpg18991":{"filename":"IMG_6.jpg","size":18991,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[108,141,128,95],"all_points_y":[136,145,192,182]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[131,185,98,43,131],"all_points_y":[109,182,247,172,109]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[100,130,160,212,183,218,127,99,56,2,43,12],"all_points_y":[68,108,91,156,180,223,292,250,277,203,174,134]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_7.jpg22888":{"filename":"IMG_7.jpg","size":22888,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[65,153,154,55],"all_points_y":[110,113,245,242]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[65,156,152,195,203,151,165,45,55,37,50,64],"all_points_y":[64,69,114,106,248,243,267,267,243,247,94,110]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[122,118,146,145],"all_points_y":[133,235,237,133]},"region_attributes":{"name":"item_rect_slim"}}],"file_attributes":{}},"IMG_8.jpg19591":{"filename":"IMG_8.jpg","size":19591,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[71,143,138,95,83,74,70,72],"all_points_y":[120,124,209,204,202,202,203,121]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[57,159,170,61],"all_points_y":[89,87,230,233]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[50,167,158,211,225,225,168,182,56,62,4,2,58],"all_points_y":[50,48,87,80,193,230,228,286,293,233,237,90,88]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_9.jpg26999":{"filename":"IMG_9.jpg","size":26999,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[72,168,154,41],"all_points_y":[68,92,235,224]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[84,190,164,200,185,156,174,43,2,1,31,72],"all_points_y":[28,58,92,72,254,235,238,230,229,159,31,68]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[84,88,144,146],"all_points_y":[106,134,134,107]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_10.jpg33625":{"filename":"IMG_10.jpg","size":33625,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[227,239,247,252,255,254,250,242,231,224,222,224],"all_points_y":[171,168,170,175,182,191,197,200,199,190,182,175]},"region_attributes":{"name":"item_circ"}},{"shape_attributes":{"name":"polygon","all_points_x":[117,120,280,286,124],"all_points_y":[123,123,115,231,237]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[107,280,281,306,317,285,295,105,127,68,63,118],"all_points_y":[72,61,117,108,243,230,257,262,233,243,123,123]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_11.jpg27698":{"filename":"IMG_11.jpg","size":27698,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[24,118,163,69],"all_points_y":[100,81,215,257]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[21,57,90,129,136,117,162,208,162,192,159,147,135,82,68,68,31,2,25],"all_points_y":[71,62,57,56,55,80,71,200,216,225,247,256,262,282,257,283,185,75,102]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[57,109,124,76],"all_points_y":[143,192,174,127]},"region_attributes":{"name":"item_rect_slim"}}],"file_attributes":{}},"IMG_12.jpg15001":{"filename":"IMG_12.jpg","size":15001,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[54,101,101,54],"all_points_y":[101,101,152,153]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[38,108,109,36],"all_points_y":[97,97,204,204]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[38,106,107,141,144,109,108,37,36,2,4,37],"all_points_y":[61,62,97,98,202,201,240,243,204,204,95,96]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_13.jpg18832":{"filename":"IMG_13.jpg","size":18832,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[110,191,130,49],"all_points_y":[66,119,214,157]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[71,109,140,220,191,224,224,174,129,105,32,50,4],"all_points_y":[35,65,23,72,118,135,168,244,211,243,190,158,128]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[85,88,150,146],"all_points_y":[156,176,162,146]},"region_attributes":{"name":"item_rect_slim"}}],"file_attributes":{}},"IMG_14.jpg25014":{"filename":"IMG_14.jpg","size":25014,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[78,151,154,75],"all_points_y":[103,103,213,213]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[76,156,151,189,197,152,157,75,76,32,37,76],"all_points_y":[64,65,103,97,214,212,251,255,212,216,94,102]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[88,89,137,136],"all_points_y":[118,187,188,118]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_15.jpg26787":{"filename":"IMG_15.jpg","size":26787,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[80,104,140,118],"all_points_y":[151,174,132,112]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[60,163,182,66],"all_points_y":[37,60,205,227]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[203,225,225,184,218,87,67,11,17,29,64,76,149,192,159,165],"all_points_y":[28,153,208,206,203,213,232,234,1,1,39,1,0,15,60,61]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_16.jpg14129":{"filename":"IMG_16.jpg","size":14129,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[48,117,120,53],"all_points_y":[89,88,189,193]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[49,115,117,144,143,120,116,118,54,54,17,14,48],"all_points_y":[52,53,90,88,187,187,189,223,227,192,195,88,89]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[56,56,100,100],"all_points_y":[111,144,143,107]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_17.jpg24868":{"filename":"IMG_17.jpg","size":24868,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[105,151,147,98],"all_points_y":[167,161,112,117]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[76,164,169,76],"all_points_y":[187,189,59,58]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[30,77,68,179,167,190,181,164,168,76,77,35],"all_points_y":[55,60,36,34,61,48,201,187,230,231,184,194]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_18.jpg12729":{"filename":"IMG_18.jpg","size":12729,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[48,60,71,80,85,85,81,72,60,49,46,49],"all_points_y":[121,130,131,125,114,104,100,94,94,100,108,122]},"region_attributes":{"name":"item_circ"}},{"shape_attributes":{"name":"polygon","all_points_x":[30,104,111,28],"all_points_y":[71,72,186,188]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[30,100,103,140,144,145,110,113,26,28,1,1,28],"all_points_y":[41,40,70,63,94,182,184,229,232,189,189,74,72]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_19.jpg12969":{"filename":"IMG_19.jpg","size":12969,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[39,93,94,38],"all_points_y":[97,99,134,135]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[34,100,105,39],"all_points_y":[81,81,182,191]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[34,99,100,135,138,103,107,43,40,2,1,1,35],"all_points_y":[45,50,84,81,182,182,216,226,191,193,114,78,80]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_20.jpg41369":{"filename":"IMG_20.jpg","size":41369,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[133,193,181,122],"all_points_y":[90,117,147,124]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[109,385,309,79],"all_points_y":[12,99,241,190]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[65,80,39,34,35,29,42,58,91,103,110,92,135,144,204,398,399,384,399,399,354,348,340,308,281],"all_points_y":[263,186,248,246,169,166,69,13,8,13,7,2,1,1,14,77,99,98,109,232,294,300,299,241,300]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_21.jpg44654":{"filename":"IMG_21.jpg","size":44654,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[109,161,165,114,110],"all_points_y":[145,132,150,163,156]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[35,288,304,102],"all_points_y":[115,44,188,250]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[35,101,98,148,342,303,304,356,362,286,307,178,2,2,31],"all_points_y":[151,282,300,299,231,185,183,215,54,42,2,1,45,84,113]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_22.jpg46718":{"filename":"IMG_22.jpg","size":46718,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[103,301,305,33],"all_points_y":[66,157,292,213]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[130,302,296,355,387,314,344,11,34,25,120,104],"all_points_y":[16,101,157,121,284,293,299,209,212,169,1,65]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[119,199,208,129],"all_points_y":[170,206,185,149]},"region_attributes":{"name":"item_rect_slim"}}],"file_attributes":{}},"IMG_23.jpg14528":{"filename":"IMG_23.jpg","size":14528,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[55,67,68,57],"all_points_y":[127,126,160,160]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[35,106,109,113,43],"all_points_y":[112,108,111,215,219]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[32,106,108,143,144,114,109,109,45,43,11,2,36],"all_points_y":[79,71,108,109,213,215,217,246,249,217,218,117,116]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_24.jpg27556":{"filename":"IMG_24.jpg","size":27556,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[202,245,253,212],"all_points_y":[93,113,92,71]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[124,265,262,123],"all_points_y":[166,161,58,64]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[116,270,260,314,313,264,270,267,111,121,121,73,78,127],"all_points_y":[204,205,160,160,62,66,21,19,25,64,68,62,167,163]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_25.jpg31143":{"filename":"IMG_25.jpg","size":31143,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[74,125,121,316,313,377,377,375,377,345,307,307,116,122,69],"all_points_y":[76,88,16,8,88,75,151,199,239,234,225,298,286,215,227]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[172,171,290,290],"all_points_y":[114,202,208,113]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[120,308,313,124],"all_points_y":[218,226,82,84]},"region_attributes":{"name":"innerbox"}}],"file_attributes":{}},"IMG_26.jpg39280":{"filename":"IMG_26.jpg","size":39280,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[175,181,246,239],"all_points_y":[118,202,197,115]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[148,265,273,152],"all_points_y":[71,61,237,238]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[131,270,267,268,286,294,272,280,146,151,92,89,150],"all_points_y":[35,21,60,64,40,250,235,296,299,236,245,63,70]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_27.jpg29843":{"filename":"IMG_27.jpg","size":29843,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[71,166,143,38],"all_points_y":[80,100,244,229]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[27,73,74,184,164,194,165,145,154,35,43,1,1],"all_points_y":[53,79,45,69,99,81,265,243,249,230,227,227,156]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[99,134,139,108],"all_points_y":[160,182,170,150]},"region_attributes":{"name":"item_rect_slim"}}],"file_attributes":{}},"IMG_28.jpg13869":{"filename":"IMG_28.jpg","size":13869,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[40,111,116,36],"all_points_y":[93,94,205,205]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[45,110,111,144,144,115,115,38,39,0,5,41],"all_points_y":[61,61,95,95,205,203,244,245,203,202,94,93]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[72,72,83,79],"all_points_y":[134,170,169,133]},"region_attributes":{"name":"item_rect_slim"}}],"file_attributes":{}},"IMG_29.jpg39275":{"filename":"IMG_29.jpg","size":39275,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[210,252,250,207],"all_points_y":[168,167,101,102]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[155,155,295,297,153],"all_points_y":[88,88,83,187,188]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[149,298,295,351,351,292,296,144,153,106,107,155],"all_points_y":[38,27,88,83,189,187,239,235,185,186,83,90]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_30.jpg21750":{"filename":"IMG_30.jpg","size":21750,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[108,134,131,115],"all_points_y":[184,184,115,115]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[72,153,158,76],"all_points_y":[99,95,209,213]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[69,154,153,193,198,156,162,75,77,39,33,72],"all_points_y":[57,53,96,93,212,207,248,253,213,219,97,101]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_31.jpg33187":{"filename":"IMG_31.jpg","size":33187,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[123,243,303,183],"all_points_y":[155,83,190,259]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[91,214,245,288,341,301,335,245,205,183,125,66,125],"all_points_y":[99,28,85,65,159,191,247,299,300,257,289,185,155]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[170,174,247,242],"all_points_y":[152,183,172,138]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_32.jpg26592":{"filename":"IMG_32.jpg","size":26592,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[104,281,357,193,168],"all_points_y":[148,63,221,299,299]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[64,195,253,281,340,397,399,357,398,195,165,65,14,104],"all_points_y":[66,1,1,64,47,165,189,221,299,299,298,299,187,149]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[163,177,259,242],"all_points_y":[140,176,135,104]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_33.jpg13396":{"filename":"IMG_33.jpg","size":13396,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[37,101,113,41],"all_points_y":[115,110,209,215]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[45,117,113,144,144,134,101,97,35,38,5,5,42],"all_points_y":[256,249,210,207,165,99,110,81,85,113,116,218,215]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[66,65,79,78],"all_points_y":[145,194,195,144]},"region_attributes":{"name":"item_rect_slim"}}],"file_attributes":{}},"IMG_34.jpg52750":{"filename":"IMG_34.jpg","size":52750,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[247,323,326,325,243,243],"all_points_y":[177,176,172,142,141,176]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[174,399,393,179],"all_points_y":[221,220,75,75]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[473,461,390,389,182,180,106,96,175,170,403,397],"all_points_y":[215,79,77,15,13,75,75,217,215,296,295,217]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_35.jpg14549":{"filename":"IMG_35.jpg","size":14549,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[58,85,98,69],"all_points_y":[158,166,122,112]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[45,119,112,43],"all_points_y":[197,193,89,90]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[45,119,115,145,143,111,108,42,43,8,7,45],"all_points_y":[235,232,193,193,89,89,56,58,90,91,197,195]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_36.jpg17102":{"filename":"IMG_36.jpg","size":17102,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[62,157,149,55],"all_points_y":[227,223,92,95]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[59,165,156,171,163,149,154,46,59,56,27,33,62],"all_points_y":[279,271,223,233,76,90,58,61,95,96,81,244,224]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[70,73,136,137],"all_points_y":[116,191,191,112]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_37.jpg25967":{"filename":"IMG_37.jpg","size":25967,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[133,301,297,126],"all_points_y":[245,231,87,90]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[45,127,118,313,295,341,350,300,308,130,134,47],"all_points_y":[91,90,5,10,89,90,219,227,298,299,242,249]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[190,197,279,265],"all_points_y":[108,173,152,98]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_38.jpg13127":{"filename":"IMG_38.jpg","size":13127,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[74,79,83,85,84,80,73,71],"all_points_y":[173,175,172,167,165,163,163,169]},"region_attributes":{"name":"item_circ"}},{"shape_attributes":{"name":"polygon","all_points_x":[44,118,108,41],"all_points_y":[224,219,115,118]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[46,119,117,145,144,140,109,105,39,40,7,6,43],"all_points_y":[265,259,220,217,155,113,115,83,85,117,117,227,223]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_39.jpg51999":{"filename":"IMG_39.jpg","size":51999,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[211,349,342,210],"all_points_y":[201,195,111,115]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[164,389,375,163],"all_points_y":[227,213,78,80]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[156,224,405,387,388,461,440,374,369,167,165,94,87,165],"all_points_y":[297,295,286,214,212,209,77,79,15,20,82,87,226,223]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_40.jpg22486":{"filename":"IMG_40.jpg","size":22486,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[75,124,137,87],"all_points_y":[190,197,116,109]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[63,170,163,62],"all_points_y":[223,221,102,105]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[69,167,170,224,224,217,162,160,61,63,7,3,63],"all_points_y":[270,267,222,222,155,93,101,51,53,105,103,225,224]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_41.jpg32999":{"filename":"IMG_41.jpg","size":32999,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[173,263,260,170],"all_points_y":[199,197,131,135]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[112,309,311,114],"all_points_y":[213,228,90,88]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[82,315,306,386,385,308,311,113,114,50,45,43,39,111],"all_points_y":[268,292,226,229,89,89,14,21,90,82,85,91,214,211]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_42.jpg25634":{"filename":"IMG_42.jpg","size":25634,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[71,147,147,65],"all_points_y":[194,194,82,84]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[68,147,145,186,189,147,149,62,69,21,31,71],"all_points_y":[233,235,195,205,83,88,44,45,85,85,201,193]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[82,84,130,128],"all_points_y":[131,168,166,130]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_43.jpg42950":{"filename":"IMG_43.jpg","size":42950,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[201,220,247,229],"all_points_y":[108,160,148,95]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[123,322,277,272,37,120],"all_points_y":[285,185,46,45,149,287]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[346,387,318,379,243,112,33,35,39,-1,1,199,284,274,278],"all_points_y":[35,203,183,225,299,299,169,157,150,115,74,0,1,45,49]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_44.jpg31593":{"filename":"IMG_44.jpg","size":31593,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[107,272,260,97],"all_points_y":[238,222,82,93]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[57,107,110,279,269,339,329,262,255,91,98,95,46],"all_points_y":[233,235,297,299,221,215,81,85,8,21,94,100,106]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[115,115,252,255],"all_points_y":[119,198,212,127]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_45.jpg27066":{"filename":"IMG_45.jpg","size":27066,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[84,110,119,91],"all_points_y":[169,180,152,143]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[24,127,198,78],"all_points_y":[80,63,205,258]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[93,197,199,224,225,199,224,224,173,127,145,91,14,26,0,1,68,77],"all_points_y":[271,233,227,212,203,206,194,137,51,64,17,17,29,79,41,55,269,257]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_46.jpg13433":{"filename":"IMG_46.jpg","size":13433,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[53,99,93,47],"all_points_y":[199,193,155,161]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[38,109,102,36],"all_points_y":[217,213,113,114]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[39,113,109,144,140,133,100,99,37,36,4,1,37],"all_points_y":[256,252,214,211,153,109,113,79,82,114,110,218,217]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_47.jpg25203":{"filename":"IMG_47.jpg","size":25203,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[28,40,47,35],"all_points_y":[144,168,165,140]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[14,149,149,30],"all_points_y":[243,234,65,49]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[183,145,208,197,148,176,96,34,35,13,9,0,1,15,7,47,83,123,123],"all_points_y":[253,233,235,73,69,35,11,1,49,1,1,81,251,242,267,272,270,267,263]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_48.jpg34931":{"filename":"IMG_48.jpg","size":34931,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[120,279,276,119],"all_points_y":[201,199,79,91]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[111,279,272,335,331,271,276,107,120,64,64,123],"all_points_y":[255,257,197,202,80,88,21,36,91,86,203,199]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[171,170,233,233],"all_points_y":[148,182,185,147]},"region_attributes":{"name":"item_rect"}}],"file_attributes":{}},"IMG_49.jpg19284":{"filename":"IMG_49.jpg","size":19284,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[126,144,145,125],"all_points_y":[192,194,141,140]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[65,148,151,66],"all_points_y":[227,225,109,105]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[61,150,147,188,191,149,153,66,68,27,25,67],"all_points_y":[269,270,221,235,107,109,67,64,107,103,231,224]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_50.jpg19178":{"filename":"IMG_50.jpg","size":19178,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[77,63,117,130],"all_points_y":[97,185,199,99]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[57,61,169,153],"all_points_y":[81,235,225,76]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[13,12,60,64,176,170,224,205,156,153,49,56],"all_points_y":[59,246,237,296,284,227,220,71,75,20,22,79]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_51.jpg12578":{"filename":"IMG_51.jpg","size":12578,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[55,62,77,70],"all_points_y":[119,145,135,117]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[36,39,107,107],"all_points_y":[76,187,180,79]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[38,106,106,141,142,108,105,39,38,1,0,35],"all_points_y":[37,44,78,77,182,181,216,222,187,190,71,74]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_52.jpg20139":{"filename":"IMG_52.jpg","size":20139,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[75,75,110,110],"all_points_y":[104,169,170,104]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[67,58,130,139],"all_points_y":[86,184,192,92]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[39,30,56,50,126,129,160,170,140,142,66,70],"all_points_y":[75,186,184,216,225,193,199,87,93,66,60,83]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_53.jpg17290":{"filename":"IMG_53.jpg","size":17290,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[85,87,116,118],"all_points_y":[129,168,173,131]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[123,119,151,156],"all_points_y":[126,168,172,129]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[59,68,164,162],"all_points_y":[85,230,231,79]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[57,160,162,211,208,164,168,68,66,67,18,6,59],"all_points_y":[30,24,79,71,250,229,286,289,282,230,235,87,85]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_54.jpg21501":{"filename":"IMG_54.jpg","size":21501,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[87,96,124,114],"all_points_y":[112,151,144,105]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[61,61,146,150],"all_points_y":[60,181,183,60]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[56,151,147,189,187,147,154,52,62,60,41,42,61],"all_points_y":[13,11,58,50,184,183,201,202,183,181,189,46,62]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_55.jpg17905":{"filename":"IMG_55.jpg","size":17905,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[123,126,160,158],"all_points_y":[147,202,200,147]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[60,68,166,159],"all_points_y":[154,236,228,142]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[53,156,160,204,214,167,174,72,69,36,27,60],"all_points_y":[109,96,140,137,226,228,275,284,238,234,156,154]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_56.jpg14750":{"filename":"IMG_56.jpg","size":14750,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[61,73,87,74],"all_points_y":[128,153,148,122]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[56,54,145,144],"all_points_y":[98,200,202,101]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[59,146,145,190,195,147,143,54,54,1,8,57],"all_points_y":[52,56,100,94,202,201,241,240,200,199,90,97]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_57.jpg32976":{"filename":"IMG_57.jpg","size":32976,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[167,177,236,226],"all_points_y":[152,177,144,124]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[125,177,304,249],"all_points_y":[163,272,209,100]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[95,221,250,296,343,305,335,275,190,176,124,67,126],"all_points_y":[105,42,100,80,180,207,272,299,300,274,300,191,163]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_58.jpg32868":{"filename":"IMG_58.jpg","size":32868,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[213,272,236,177],"all_points_y":[197,166,107,136]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[107,110,308,306],"all_points_y":[87,213,218,76]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[104,304,304,382,387,310,316,80,111,37,36,106],"all_points_y":[19,2,74,74,217,218,283,269,213,218,85,88]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_59.jpg30456":{"filename":"IMG_59.jpg","size":30456,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[155,154,286,288],"all_points_y":[100,123,133,109]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[118,134,319,308],"all_points_y":[71,205,194,60]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[103,230,314,309,377,389,316,334,124,134,63,52,117],"all_points_y":[10,1,0,58,45,199,193,259,269,202,216,72,73]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_60.jpg18946":{"filename":"IMG_60.jpg","size":18946,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[116,104,118,132],"all_points_y":[134,184,188,140]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[41,102,181,119],"all_points_y":[181,246,176,109]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[4,84,116,146,203,182,212,132,101,65,4,40],"all_points_y":[146,73,107,86,148,175,212,286,247,282,215,182]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_61.jpg24194":{"filename":"IMG_61.jpg","size":24194,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[109,109,131,131],"all_points_y":[134,157,157,134]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[76,76,150,154],"all_points_y":[85,191,195,86]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[33,38,75,72,149,150,187,195,155,158,74,78],"all_points_y":[81,200,190,225,231,196,202,86,87,46,46,84]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_62.jpg20584":{"filename":"IMG_62.jpg","size":20584,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[105,140,124,89],"all_points_y":[178,170,100,110]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[75,158,156,77],"all_points_y":[80,80,194,191]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[76,158,157,199,197,154,156,73,78,38,34,78],"all_points_y":[38,37,80,78,198,192,231,226,190,196,76,80]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_63.jpg12702":{"filename":"IMG_63.jpg","size":12702,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[66,66,90,89],"all_points_y":[144,184,184,145]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[40,38,112,107],"all_points_y":[108,211,209,106]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[40,105,105,139,144,144,113,113,40,40,1,6,42],"all_points_y":[73,73,105,104,156,210,209,247,250,211,215,100,107]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_64.jpg27386":{"filename":"IMG_64.jpg","size":27386,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[84,89,136,132],"all_points_y":[92,120,110,85]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[53,161,145,41],"all_points_y":[63,75,226,227]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[28,21,42,38,164,145,193,211,160,185,51,55],"all_points_y":[35,258,228,261,257,224,228,80,76,62,40,63]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_65.jpg14005":{"filename":"IMG_65.jpg","size":14005,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[67,61,80,85],"all_points_y":[153,188,193,157]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[38,41,113,98],"all_points_y":[110,202,196,106]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[39,95,99,126,144,144,114,119,45,43,5,7,39],"all_points_y":[85,84,107,106,184,192,194,233,241,201,206,111,110]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_66.jpg16631":{"filename":"IMG_66.jpg","size":16631,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[119,129,166,156],"all_points_y":[177,197,180,162]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[59,64,182,178],"all_points_y":[123,223,219,114]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[54,175,178,215,222,180,187,66,64,10,6,58],"all_points_y":[67,58,114,115,207,214,272,278,222,226,126,122]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_67.jpg20229":{"filename":"IMG_67.jpg","size":20229,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[108,108,142,143],"all_points_y":[113,200,201,116]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[64,163,168,63],"all_points_y":[79,79,230,232]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[62,168,164,216,224,168,170,62,63,12,23,65],"all_points_y":[22,23,79,76,231,228,285,287,232,241,59,80]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_68.jpg33373":{"filename":"IMG_68.jpg","size":33373,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[165,154,210,222],"all_points_y":[145,188,205,163]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[109,117,285,280],"all_points_y":[101,220,213,92]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[96,282,280,344,349,286,294,109,118,56,48,108],"all_points_y":[41,35,92,88,212,212,275,279,218,229,97,99]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_69.jpg18623":{"filename":"IMG_69.jpg","size":18623,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[76,127,147,92],"all_points_y":[125,167,148,100]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[40,133,192,100],"all_points_y":[137,204,124,56]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[16,67,96,128,224,224,193,225,224,179,135,109,2,41],"all_points_y":[115,44,58,12,74,84,124,146,171,235,206,252,172,136]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_70.jpg19499":{"filename":"IMG_70.jpg","size":19499,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[78,88,118,110],"all_points_y":[116,191,186,114]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[70,70,162,165],"all_points_y":[66,197,204,68]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[72,168,166,218,212,162,164,64,70,19,20,70],"all_points_y":[16,18,69,68,209,202,247,242,196,201,61,68]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_71.jpg45811":{"filename":"IMG_71.jpg","size":45811,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[187,237,252,200],"all_points_y":[157,214,200,145]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[130,198,299,234],"all_points_y":[221,72,118,270]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[157,76,127,124,238,243,247,328,302,298,324,210,200],"all_points_y":[42,211,222,225,278,292,293,107,117,115,94,41,75]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_72.jpg15553":{"filename":"IMG_72.jpg","size":15553,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[41,41,94,93],"all_points_y":[109,175,174,110]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[35,36,104,103],"all_points_y":[94,200,195,95]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[36,104,104,133,133,103,102,38,38,0,1,35],"all_points_y":[57,61,96,97,194,196,228,234,200,202,93,94]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_73.jpg19455":{"filename":"IMG_73.jpg","size":19455,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[109,104,131,136],"all_points_y":[140,176,179,142]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[68,70,168,164],"all_points_y":[102,237,232,96]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[66,165,162,208,214,164,173,67,70,23,23,69],"all_points_y":[54,48,102,93,241,230,283,286,234,245,97,103]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_74.jpg13680":{"filename":"IMG_74.jpg","size":13680,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[49,49,70,70],"all_points_y":[100,131,131,101]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[38,41,112,112],"all_points_y":[76,186,182,77]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[36,108,110,142,143,110,111,43,42,3,0,1,37],"all_points_y":[37,42,80,78,181,182,218,224,186,190,108,74,76]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_75.jpg20669":{"filename":"IMG_75.jpg","size":20669,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[111,112,133,131],"all_points_y":[109,167,166,110]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[72,107,182,148],"all_points_y":[108,193,164,76]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[108,123,197,180,222,189,147,129,60,73,29,67],"all_points_y":[194,232,202,164,149,62,77,48,72,106,123,210]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_76.jpg27147":{"filename":"IMG_76.jpg","size":27147,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[174,178,238,234],"all_points_y":[108,130,120,97]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[186,188,247,243],"all_points_y":[139,160,150,128]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[135,138,311,305],"all_points_y":[82,201,199,85]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[135,310,305,362,365,308,331,133,140,78,75,87,136],"all_points_y":[24,27,86,86,193,198,243,250,199,207,84,79,87]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_77.jpg45935":{"filename":"IMG_77.jpg","size":45935,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[154,192,239,193],"all_points_y":[164,246,229,147]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[71,92,327,278],"all_points_y":[121,269,220,77]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[62,263,279,304,344,370,327,368,368,112,62,93,50,35,74],"all_points_y":[54,13,77,36,117,209,218,229,234,298,298,265,279,92,119]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_78.jpg18381":{"filename":"IMG_78.jpg","size":18381,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[101,96,98,102,111,124,134,141,144,144,141,136,127,119,112,106],"all_points_y":[127,136,146,155,162,164,161,154,145,136,128,123,118,118,119,121]},"region_attributes":{"name":"item_circ"}},{"shape_attributes":{"name":"polygon","all_points_x":[65,57,157,165],"all_points_y":[79,218,224,80]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[64,168,163,211,208,156,164,44,58,3,20,67],"all_points_y":[26,29,83,72,232,221,271,267,214,222,62,78]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_79.jpg18012":{"filename":"IMG_79.jpg","size":18012,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[103,130,144,116],"all_points_y":[127,172,163,120]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[47,93,188,140],"all_points_y":[116,225,183,76]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[42,93,115,208,184,224,225,188,139,120,36,48,0,2],"all_points_y":[250,226,276,232,183,166,140,58,79,48,79,114,135,156]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_80.jpg19043":{"filename":"IMG_80.jpg","size":19043,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[111,104,132,142],"all_points_y":[102,188,191,105]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[66,73,155,155],"all_points_y":[87,206,204,81]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[59,159,155,198,197,155,162,64,73,56,48,67],"all_points_y":[46,39,82,76,212,203,229,234,203,220,80,89]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_81.jpg18816":{"filename":"IMG_81.jpg","size":18816,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[88,84,124,126],"all_points_y":[121,188,190,123]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[77,68,176,175],"all_points_y":[83,222,228,87]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[73,186,174,224,224,176,189,58,69,10,26,78],"all_points_y":[42,49,90,86,230,227,282,275,219,219,77,84]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_82.jpg55185":{"filename":"IMG_82.jpg","size":55185,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[240,241,310,308],"all_points_y":[137,160,154,131]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[153,154,376,370],"all_points_y":[71,220,218,61]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[155,154,88,89,155,155,380,375,456,450,369,368],"all_points_y":[4,74,81,218,218,292,296,213,215,68,68,1]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_83.jpg34933":{"filename":"IMG_83.jpg","size":34933,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[202,191,251,261],"all_points_y":[120,173,184,131]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[154,151,305,307],"all_points_y":[94,200,204,87]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[100,99,149,140,307,302,361,362,301,312,137,155],"all_points_y":[91,199,197,249,264,201,208,85,93,31,45,93]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_84.jpg19315":{"filename":"IMG_84.jpg","size":19315,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[77,100,149,127],"all_points_y":[131,159,115,90]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[53,59,157,163],"all_points_y":[64,203,208,66]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[41,172,161,221,211,153,162,47,61,7,1,1,54],"all_points_y":[9,10,67,67,213,207,250,243,203,211,124,63,65]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_85.jpg28749":{"filename":"IMG_85.jpg","size":28749,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[54,64,128,125,117,51],"all_points_y":[133,128,153,170,172,148]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[99,1,1,127,187],"all_points_y":[63,127,137,284,196]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[6,130,133,98,138,215,188,224,225,191,149,126,141,133,1,1],"all_points_y":[102,29,35,63,39,163,193,198,229,300,300,284,299,299,147,117]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_86.jpg14521":{"filename":"IMG_86.jpg","size":14521,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[37,46,82,72],"all_points_y":[104,180,174,100]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[22,32,103,97],"all_points_y":[84,195,189,80]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[22,95,96,128,136,101,104,36,33,1,1,26],"all_points_y":[47,45,82,81,183,187,222,230,191,195,85,85]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_87.jpg24135":{"filename":"IMG_87.jpg","size":24135,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[61,62,150,153],"all_points_y":[104,189,189,101]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[56,52,164,168],"all_points_y":[71,226,224,72]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[52,178,165,224,220,159,169,48,53,3,8,57],"all_points_y":[10,15,72,67,237,222,283,288,226,243,49,71]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_88.jpg17219":{"filename":"IMG_88.jpg","size":17219,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[116,94,128,150],"all_points_y":[134,175,194,154]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[66,76,171,160],"all_points_y":[71,202,196,68]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[59,164,159,207,220,167,180,77,76,25,16,67],"all_points_y":[22,18,70,64,193,196,247,255,199,209,71,74]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_89.jpg13203":{"filename":"IMG_89.jpg","size":13203,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[46,52,79,71],"all_points_y":[145,156,137,127]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[31,35,106,100],"all_points_y":[84,192,185,85]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[31,101,99,132,138,102,106,38,37,1,0,32],"all_points_y":[44,48,86,85,185,186,222,228,191,198,76,84]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_90.jpg18293":{"filename":"IMG_90.jpg","size":18293,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[82,75,115,123],"all_points_y":[122,162,171,129]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[72,142,144,70],"all_points_y":[180,177,78,77]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[36,39,73,71,146,141,175,178,144,147,66,70],"all_points_y":[73,182,179,212,211,174,180,71,78,43,42,76]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_91.jpg18187":{"filename":"IMG_91.jpg","size":18187,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[65,67,96,93],"all_points_y":[158,225,223,158]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[58,61,152,146],"all_points_y":[111,238,233,106]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[48,147,145,186,194,149,158,59,62,17,15,59],"all_points_y":[74,68,110,100,240,231,281,286,235,248,108,112]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_92.jpg48632":{"filename":"IMG_92.jpg","size":48632,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[193,203,214,219,221,217,209,203,198,194,193],"all_points_y":[191,200,199,191,182,175,172,171,174,179,185]},"region_attributes":{"name":"item_circ"}},{"shape_attributes":{"name":"polygon","all_points_x":[111,204,326,216],"all_points_y":[124,284,204,53]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[80,198,214,212,361,357,333,337,196,206,200,131,47,112,113],"all_points_y":[84,7,52,7,206,211,211,200,295,295,299,300,132,123,115]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_93.jpg30568":{"filename":"IMG_93.jpg","size":30568,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[165,236,215,141],"all_points_y":[116,141,201,175]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[132,141,295,285],"all_points_y":[87,224,205,76]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[120,297,283,324,336,290,306,148,141,65,54,131],"all_points_y":[10,7,77,77,195,203,277,300,222,233,89,86]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_94.jpg38226":{"filename":"IMG_94.jpg","size":38226,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[181,170,264,276],"all_points_y":[114,178,195,125]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[141,131,293,304],"all_points_y":[86,196,218,97]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[88,79,132,116,289,291,349,363,301,313,131,144],"all_points_y":[76,194,195,244,272,214,226,97,99,49,32,84]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_95.jpg18135":{"filename":"IMG_95.jpg","size":18135,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[93,112,131,112],"all_points_y":[171,180,132,125]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[67,73,161,156],"all_points_y":[110,235,229,104]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[63,155,156,199,203,158,168,72,73,31,24,68],"all_points_y":[67,61,108,101,239,226,275,285,235,246,109,109]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_96.jpg30610":{"filename":"IMG_96.jpg","size":30610,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[199,206,249,242],"all_points_y":[153,170,150,136]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[123,179,295,238],"all_points_y":[152,253,184,83]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[89,207,238,279,332,295,328,225,203,178,127,70,122],"all_points_y":[100,30,84,63,155,184,239,300,299,252,280,184,153]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_97.jpg59317":{"filename":"IMG_97.jpg","size":59317,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[247,244,323,327],"all_points_y":[97,181,183,95]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[180,184,394,387],"all_points_y":[76,218,212,57]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[180,180,120,123,184,185,402,395,477,467,385,384,294],"all_points_y":[11,78,85,217,216,287,292,209,211,55,64,3,3]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_98.jpg44280":{"filename":"IMG_98.jpg","size":44280,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[146,164,197,177],"all_points_y":[153,223,213,145]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[23,113,333,295],"all_points_y":[189,62,121,297]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[143,351,334,375,380,340,300,271,2,1,24,1,1,82,110],"all_points_y":[11,53,120,53,148,287,299,298,195,186,189,171,82,2,62]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_99.jpg12953":{"filename":"IMG_99.jpg","size":12953,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[38,38,80,80],"all_points_y":[100,118,119,100]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[32,35,104,103],"all_points_y":[88,195,191,89]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[32,101,102,138,139,103,102,36,35,1,1,33],"all_points_y":[51,57,91,88,191,190,225,231,194,197,84,88]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_100.jpg44961":{"filename":"IMG_100.jpg","size":44961,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[169,184,242,224],"all_points_y":[113,100,174,187]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[101,330,288,66],"all_points_y":[44,83,260,193]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[80,367,341,291,296,291,109,9,65,3,2,36,99,59],"all_points_y":[1,43,81,282,275,299,299,261,192,186,160,11,45,1]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_101.jpg19203":{"filename":"IMG_101.jpg","size":19203,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[98,142,144,99],"all_points_y":[106,105,181,181]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[165,167,68,62],"all_points_y":[85,227,230,84]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[223,224,219,166,171,60,64,7,14,67,62,176,167,225],"all_points_y":[201,202,77,85,27,28,82,80,240,231,280,283,226,237]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_102.jpg44827":{"filename":"IMG_102.jpg","size":44827,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[181,270,264,176],"all_points_y":[117,129,182,167]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[272,277,146,142],"all_points_y":[201,100,99,190]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[267,271,321,325,278,281,139,145,98,96,141,130],"all_points_y":[248,198,206,105,106,51,53,97,97,187,189,236]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_103.jpg43859":{"filename":"IMG_103.jpg","size":43859,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[148,183,177,141],"all_points_y":[100,105,146,138]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[97,266,230,99],"all_points_y":[44,58,235,239]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[272,357,293,230,243,92,99,97,79,72,100,94,297,96,103,272,276],"all_points_y":[28,49,269,232,293,299,239,239,299,13,11,41,60,41,10,28,36]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_104.jpg20819":{"filename":"IMG_104.jpg","size":20819,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[100,131,123,94],"all_points_y":[92,93,169,166]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[139,139,65,62],"all_points_y":[74,179,181,71]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[63,141,138,174,173,139,138,56,63,34,29,60],"all_points_y":[28,29,72,70,188,180,217,216,181,188,61,70]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_105.jpg23594":{"filename":"IMG_105.jpg","size":23594,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[123,140,99,82],"all_points_y":[124,182,189,132]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[154,162,80,73],"all_points_y":[92,205,210,95]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[207,159,165,80,80,35,33,75,69,154,152,194],"all_points_y":[203,205,248,251,210,215,91,94,56,52,91,83]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_106.jpg18173":{"filename":"IMG_106.jpg","size":18173,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[81,130,129,84],"all_points_y":[120,119,190,192]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[165,173,63,63],"all_points_y":[78,222,224,79]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[175,166,221,225,225,173,184,56,65,6,9,64,57],"all_points_y":[39,77,72,119,220,222,278,282,223,226,76,80,38]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_107.jpg24766":{"filename":"IMG_107.jpg","size":24766,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[77,129,151,100],"all_points_y":[143,129,206,223]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[154,163,65,64],"all_points_y":[102,238,242,107]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[155,155,199,212,163,175,56,64,46,45,61,59],"all_points_y":[56,104,93,238,237,264,273,241,254,90,107,57]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_108.jpg27807":{"filename":"IMG_108.jpg","size":27807,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[103,113,117,118,118,115,109,103,99,97,95,94,94,95,98,99,100],"all_points_y":[127,128,131,136,144,149,152,152,152,150,147,145,142,138,131,131,129]},"region_attributes":{"name":"item_circ"}},{"shape_attributes":{"name":"polygon","all_points_x":[120,181,72,22],"all_points_y":[73,212,261,103]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[122,169,225,224,182,208,168,163,153,97,90,82,76,73,73,57,1,1,22,6,78,106,126,130],"all_points_y":[72,57,176,194,214,223,247,251,257,279,281,283,286,283,264,285,115,86,100,70,49,46,41,41]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_109.jpg22710":{"filename":"IMG_109.jpg","size":22710,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[77,126,125,81],"all_points_y":[145,145,208,206]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[153,157,64,68],"all_points_y":[96,222,223,96]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[202,157,166,54,66,48,50,67,65,157,153,195],"all_points_y":[225,223,244,246,221,231,79,91,50,50,97,86]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_110.jpg17364":{"filename":"IMG_110.jpg","size":17364,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[77,104,153,125],"all_points_y":[110,94,180,196]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[168,163,65,70],"all_points_y":[74,208,204,66]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[176,167,218,216,164,167,62,67,11,16,70,70],"all_points_y":[20,73,74,210,207,261,258,203,205,61,68,14]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_111.jpg15269":{"filename":"IMG_111.jpg","size":15269,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[48,95,106,57],"all_points_y":[116,109,175,184]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[40,107,110,41],"all_points_y":[98,100,198,199]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[43,105,107,137,141,105,109,41,41,3,8,41],"all_points_y":[70,70,98,100,195,197,237,236,198,200,97,98]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_112.jpg21597":{"filename":"IMG_112.jpg","size":21597,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[83,140,141,83],"all_points_y":[123,123,196,196]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[159,161,66,71],"all_points_y":[93,221,223,92]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[166,160,201,204,160,165,65,66,21,27,71,68],"all_points_y":[49,95,84,225,221,268,266,217,226,83,92,46]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_113.jpg19243":{"filename":"IMG_113.jpg","size":19243,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[137,112,137,157],"all_points_y":[117,138,163,132]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[170,170,77,83],"all_points_y":[112,242,240,110]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[81,175,168,214,217,169,178,69,79,36,44,83],"all_points_y":[66,65,113,104,251,240,290,287,236,247,102,111]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_114.jpg17093":{"filename":"IMG_114.jpg","size":17093,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[88,97,134,123],"all_points_y":[103,157,151,97]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[151,151,159,80,74],"all_points_y":[59,58,174,180,61]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[66,153,151,192,202,159,162,85,79,47,40,73],"all_points_y":[13,13,59,54,172,174,216,224,178,188,48,62]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_115.jpg29392":{"filename":"IMG_115.jpg","size":29392,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[116,117,166,166,117],"all_points_y":[128,128,128,148,149]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[205,77,11,143],"all_points_y":[148,243,143,64]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[174,224,224,207,206,224,224,72,76,2,10,0,1,131,143,145],"all_points_y":[32,99,136,145,148,146,154,267,247,133,142,129,78,8,63,66]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_116.jpg18564":{"filename":"IMG_116.jpg","size":18564,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[102,148,133,92],"all_points_y":[94,98,203,199]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[159,163,62,57],"all_points_y":[72,226,226,77]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[160,160,162,210,206,159,170,62,65,13,0,55,54],"all_points_y":[15,16,73,65,247,223,283,278,226,228,79,77,21]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_117.jpg17005":{"filename":"IMG_117.jpg","size":17005,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[102,113,110,98],"all_points_y":[131,132,173,170]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[143,151,59,56],"all_points_y":[98,198,202,100]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[55,140,142,186,198,151,151,64,61,3,8,56],"all_points_y":[56,55,96,88,194,198,237,244,203,203,101,101]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_118.jpg20674":{"filename":"IMG_118.jpg","size":20674,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[93,142,142,91],"all_points_y":[131,133,209,208]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[70,163,171,66],"all_points_y":[92,91,240,242]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[67,166,163,213,225,224,170,171,69,66,19,27,68],"all_points_y":[40,39,92,89,218,237,239,298,300,242,252,74,93]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_119.jpg23138":{"filename":"IMG_119.jpg","size":23138,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[114,134,118,99],"all_points_y":[129,132,204,198]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[150,153,70,70],"all_points_y":[105,219,217,104]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[71,153,150,186,193,152,156,67,71,42,38,68,73],"all_points_y":[76,76,105,98,224,217,258,258,216,227,94,103,102]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_120.jpg33838":{"filename":"IMG_120.jpg","size":33838,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[158,227,222,153],"all_points_y":[97,103,183,185]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[127,233,226,117],"all_points_y":[88,94,244,240]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[240,235,283,277,225,231,109,119,57,75,128,128],"all_points_y":[43,92,90,253,240,292,295,241,245,77,86,36]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_121.jpg50287":{"filename":"IMG_121.jpg","size":50287,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[254,278,280,254],"all_points_y":[139,139,203,206]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[162,367,382,165],"all_points_y":[102,89,227,246]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[161,360,369,434,450,378,389,340,169,165,90,90,162],"all_points_y":[40,30,91,87,221,228,299,299,300,244,246,105,104]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_122.jpg55699":{"filename":"IMG_122.jpg","size":55699,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[209,360,363,213],"all_points_y":[101,98,212,223]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[172,407,403,173],"all_points_y":[82,86,235,240]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[173,401,404,474,474,403,400,175,175,93,89,172],"all_points_y":[5,12,86,90,235,235,299,299,240,242,85,83]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_123.jpg43650":{"filename":"IMG_123.jpg","size":43650,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[152,173,202,178],"all_points_y":[111,196,189,104]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[98,226,283,118],"all_points_y":[81,53,214,258]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[74,156,219,221,225,233,238,324,293,298,96,122,68,24,24,96,101],"all_points_y":[21,4,4,50,50,3,3,228,246,217,267,291,300,298,58,82,80]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_124.jpg13800":{"filename":"IMG_124.jpg","size":13800,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[58,58,86,86],"all_points_y":[132,165,164,132]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[109,110,45,43],"all_points_y":[88,185,189,83]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[43,107,108,138,139,110,110,44,45,9,8,43],"all_points_y":[50,55,87,88,182,184,218,223,188,191,82,86]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_125.jpg21059":{"filename":"IMG_125.jpg","size":21059,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[80,136,137,83],"all_points_y":[127,127,190,191]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[60,156,168,63],"all_points_y":[78,77,224,235]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[204,224,167,173,66,65,17,15,57,53,154,156],"all_points_y":[73,220,222,280,291,231,246,59,82,23,21,75]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_126.jpg28377":{"filename":"IMG_126.jpg","size":28377,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[116,145,130,99],"all_points_y":[114,123,194,187]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[36,139,190,89],"all_points_y":[73,50,193,237]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[44,91,101,214,224,224,167,140,152,30,38,1,1],"all_points_y":[269,237,240,192,190,182,14,48,4,24,72,65,144]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_127.jpg25215":{"filename":"IMG_127.jpg","size":25215,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[158,192,217,182],"all_points_y":[156,145,206,222]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[229,269,131,104],"all_points_y":[90,234,265,114]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[104,210,228,293,342,270,288,289,137,133,56,33,105],"all_points_y":[73,49,90,71,219,236,296,299,298,267,291,111,113]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_128.jpg19327":{"filename":"IMG_128.jpg","size":19327,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[91,164,162,96],"all_points_y":[80,78,189,191]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[73,168,166,80],"all_points_y":[73,73,201,201]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[67,173,169,189,186,169,175,72,78,67,60,73],"all_points_y":[25,21,72,63,220,202,237,233,201,217,66,70]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_129.jpg15895":{"filename":"IMG_129.jpg","size":15895,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[100,125,137,111],"all_points_y":[164,151,173,184]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[162,170,64,55],"all_points_y":[97,218,221,101]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[47,160,163,224,225,169,167,73,67,8,2,1,55],"all_points_y":[40,35,98,87,228,217,250,257,224,225,157,104,101]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_130.jpg24806":{"filename":"IMG_130.jpg","size":24806,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[96,106,121,112],"all_points_y":[123,144,135,117]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[34,167,219,58],"all_points_y":[38,28,225,265]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[25,61,52,107,210,208,224,224,164,179,26,35,8,1,2],"all_points_y":[298,264,300,298,277,272,264,19,28,1,0,36,1,0,169]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_131.jpg31491":{"filename":"IMG_131.jpg","size":31491,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[154,238,239,155],"all_points_y":[139,140,201,198]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[147,320,323,145],"all_points_y":[85,80,208,204]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[89,90,147,138,330,320,388,387,319,333,133,148],"all_points_y":[74,208,202,260,267,204,214,74,82,23,28,83]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_132.jpg33547":{"filename":"IMG_132.jpg","size":33547,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[236,234,261,265],"all_points_y":[108,188,190,109]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[188,325,326,186],"all_points_y":[100,98,197,196]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[182,335,324,378,377,326,334,181,187,147,147,187],"all_points_y":[59,57,99,94,200,197,243,238,194,198,93,97]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_133.jpg17878":{"filename":"IMG_133.jpg","size":17878,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[95,160,142,81],"all_points_y":[102,119,184,170]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[58,170,160,63],"all_points_y":[65,69,209,206]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[45,178,172,223,214,160,165,52,65,11,1,57],"all_points_y":[17,15,69,67,215,210,254,247,206,210,66,66]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_134.jpg18332":{"filename":"IMG_134.jpg","size":18332,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[72,137,138,69],"all_points_y":[89,91,195,196]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[63,149,153,65],"all_points_y":[75,76,197,199]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[56,155,150,162,167,153,161,57,64,45,46,63],"all_points_y":[38,40,75,56,200,197,241,245,199,204,56,75]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_135.jpg54041":{"filename":"IMG_135.jpg","size":54041,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[233,270,287,250],"all_points_y":[149,116,137,169]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[154,364,378,152],"all_points_y":[64,59,199,212]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[360,367,435,452,373,385,152,150,74,82,154,157],"all_points_y":[0,60,61,192,201,276,289,210,213,70,67,4]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_136.jpg13267":{"filename":"IMG_136.jpg","size":13267,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[71,53,71,91],"all_points_y":[149,178,189,160]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[33,95,103,36],"all_points_y":[115,112,206,211]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[94,96,125,136,103,106,37,35,1,3,32,35],"all_points_y":[83,111,111,204,206,242,250,211,214,108,113,83]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_137.jpg21402":{"filename":"IMG_137.jpg","size":21402,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[70,71,115,115,71],"all_points_y":[108,108,106,169,170]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[58,131,127,164,161,126,124,51,55,30,29,57],"all_points_y":[60,66,100,102,206,201,232,228,196,205,90,96]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[129,128,126,55,55,125],"all_points_y":[108,102,201,196,94,99]},"region_attributes":{"name":"innerbox"}}],"file_attributes":{}},"IMG_138.jpg26859":{"filename":"IMG_138.jpg","size":26859,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[128,134,157,150],"all_points_y":[198,221,217,194]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[22,142,224,224,106],"all_points_y":[100,56,202,227,300]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[148,138,187,192,223,223,225,224,199,69,2,2,18,21,2,2,37],"all_points_y":[1,54,35,35,80,196,228,286,300,300,139,87,100,99,59,51,36]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_139.jpg13632":{"filename":"IMG_139.jpg","size":13632,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[41,74,77,43],"all_points_y":[99,99,143,146]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[37,37,101,99,39],"all_points_y":[78,77,81,175,180]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[34,101,99,132,131,99,103,40,39,1,1,34],"all_points_y":[42,48,79,79,178,176,206,214,182,185,74,77]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_140.jpg24619":{"filename":"IMG_140.jpg","size":24619,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[61,132,141,68],"all_points_y":[116,111,201,207]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[48,142,149,51],"all_points_y":[99,96,231,236]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[42,143,142,186,198,147,156,36,50,27,26,47],"all_points_y":[52,50,95,86,234,229,257,266,235,250,84,100]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_141.jpg35744":{"filename":"IMG_141.jpg","size":35744,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[193,125,138,207],"all_points_y":[141,216,227,148]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[245,245,297,83,42],"all_points_y":[75,76,210,282,133]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[12,34,73,65,101,333,294,295,343,294,284,252,247,240],"all_points_y":[69,117,284,300,297,222,212,208,203,67,49,76,74,3]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_142.jpg14503":{"filename":"IMG_142.jpg","size":14503,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[61,91,91,63],"all_points_y":[129,129,184,182]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[43,112,118,43],"all_points_y":[103,100,201,205]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[42,107,108,144,143,145,119,117,45,45,8,9,41],"all_points_y":[70,66,98,100,132,201,200,238,243,205,208,102,102]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_143.jpg52315":{"filename":"IMG_143.jpg","size":52315,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[318,343,352,356,359,354,351,347,336,324,309,304,291,288,288,287,287,293,300,304],"all_points_y":[85,93,101,112,121,137,143,153,156,158,156,154,141,137,127,120,108,98,93,89]},"region_attributes":{"name":"item_circ"}},{"shape_attributes":{"name":"polygon","all_points_x":[377,377,394,172,164],"all_points_y":[41,40,179,198,53]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[446,468,392,403,173,175,98,91,165,164,370,377],"all_points_y":[35,171,176,254,278,198,207,63,53,1,1,37]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_144.jpg26901":{"filename":"IMG_144.jpg","size":26901,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[194,174,195,216],"all_points_y":[134,198,205,142]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[283,183,106,210],"all_points_y":[171,260,169,84]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[250,314,285,328,236,216,186,143,65,107,63,172,210,210],"all_points_y":[57,134,170,217,299,299,259,297,204,171,125,34,83,79]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_145.jpg26898":{"filename":"IMG_145.jpg","size":26898,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[106,127,134,107],"all_points_y":[111,111,187,190]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[155,151,59,67],"all_points_y":[90,215,210,83]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[67,161,157,190,194,152,157,52,61,17,28,67],"all_points_y":[38,47,90,78,224,212,245,241,208,213,66,84]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_146.jpg19663":{"filename":"IMG_146.jpg","size":19663,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[82,137,158,101],"all_points_y":[148,131,198,216]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[68,167,168,65],"all_points_y":[94,92,231,237]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[65,170,167,219,219,168,175,60,65,17,19,69],"all_points_y":[41,43,95,85,242,230,289,290,235,244,87,93]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_147.jpg20190":{"filename":"IMG_147.jpg","size":20190,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[88,151,148,87],"all_points_y":[82,85,160,158]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[58,165,156,62],"all_points_y":[70,69,221,216]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[59,167,166,212,195,155,158,58,63,14,5,57],"all_points_y":[13,12,72,66,241,218,273,268,214,218,68,69]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_148.jpg28133":{"filename":"IMG_148.jpg","size":28133,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[278,303,295,271],"all_points_y":[74,80,109,102]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[160,314,312,162],"all_points_y":[72,64,199,201]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[111,160,156,317,315,387,387,315,319,164,165,113],"all_points_y":[76,72,1,1,66,63,198,198,271,269,198,196]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_149.jpg19729":{"filename":"IMG_149.jpg","size":19729,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[100,123,111,89],"all_points_y":[108,113,188,185]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[68,69,165,162,69],"all_points_y":[70,71,69,201,202]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[56,170,165,166,182,179,162,166,65,71,26,21,66],"all_points_y":[45,41,67,69,55,217,199,252,250,201,211,65,69]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}}},"_via_attributes":{"region":{"name":{"type":"dropdown","description":"","options":{"outerbox":"","innerbox":"","item_sq":"","item_rect":"","item_rect_slim":"","item_circ":""},"default_options":{}}},"file":{}}} \ No newline at end of file diff --git a/mask_rcnn/dataset/VAL-MINI.json b/mask_rcnn/dataset/VAL-MINI.json new file mode 100644 index 00000000..e6284959 --- /dev/null +++ b/mask_rcnn/dataset/VAL-MINI.json @@ -0,0 +1 @@ +{"_via_settings":{"ui":{"annotation_editor_height":25,"annotation_editor_fontsize":0.8,"leftsidebar_width":18,"image_grid":{"img_height":80,"rshape_fill":"none","rshape_fill_opacity":0.3,"rshape_stroke":"yellow","rshape_stroke_width":2,"show_region_shape":true,"show_image_policy":"all"},"image":{"region_label":"name","region_color":"__via_default_region_color__","region_label_font":"10px Sans","on_image_annotation_editor_placement":"NEAR_REGION"}},"core":{"buffer_size":"18","filepath":{},"default_filepath":"val/"},"project":{"name":"VAL"}},"_via_img_metadata":{"IMG_150.jpg31278":{"filename":"IMG_150.jpg","size":31278,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[138,139,226,226],"all_points_y":[125,185,187,124]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[127,129,310,304],"all_points_y":[66,192,193,62]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[70,70,130,121,316,305,373,368,301,315,116,128],"all_points_y":[58,195,188,248,249,189,195,53,65,3,10,68]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_151.jpg16222":{"filename":"IMG_151.jpg","size":16222,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[86,87,122,120],"all_points_y":[114,161,161,113]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[56,56,169,166],"all_points_y":[72,196,196,71]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[3,1,59,54,169,166,223,224,159,156,62,58],"all_points_y":[58,200,196,254,254,193,195,67,72,38,40,74]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_152.jpg13817":{"filename":"IMG_152.jpg","size":13817,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[71,73,93,90],"all_points_y":[92,119,116,91]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[42,43,114,113],"all_points_y":[75,188,183,77]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[5,7,46,45,111,112,140,138,110,109,41,42],"all_points_y":[74,191,187,223,216,180,180,82,81,46,40,78]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_153.jpg14604":{"filename":"IMG_153.jpg","size":14604,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[48,50,90,86],"all_points_y":[98,150,150,98]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[41,42,122,112],"all_points_y":[77,180,179,74]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[9,6,44,44,119,116,144,142,110,105,43,41],"all_points_y":[78,182,180,220,216,177,176,76,76,45,46,77]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_154.jpg13797":{"filename":"IMG_154.jpg","size":13797,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[28,32,105,104],"all_points_y":[114,216,217,112]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[36,101,100,134,139,102,104,31,30,1,4,35],"all_points_y":[244,247,214,218,114,115,75,77,115,115,213,214]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[60,74,83,84,79,62,55,52,52],"all_points_y":[146,148,154,165,170,169,165,155,149]},"region_attributes":{"name":"item_circ"}}],"file_attributes":{}},"IMG_155.jpg29096":{"filename":"IMG_155.jpg","size":29096,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[170,153,250,266],"all_points_y":[98,188,203,114]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[115,129,304,285],"all_points_y":[84,237,215,64]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[44,54,132,135,268,315,300,354,335,285,277,106,114],"all_points_y":[96,239,231,298,300,296,206,200,69,68,1,9,86]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_156.jpg40095":{"filename":"IMG_156.jpg","size":40095,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[160,203,224,174],"all_points_y":[123,186,170,112]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[112,105,312,306],"all_points_y":[90,227,238,97]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[91,78,108,74,211,337,306,344,348,340,303,308,118,112],"all_points_y":[70,233,221,256,263,264,230,246,138,81,104,33,28,92]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_157.jpg29987":{"filename":"IMG_157.jpg","size":29987,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[250,261,284,278],"all_points_y":[114,176,175,109]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[170,172,319,317],"all_points_y":[79,207,203,75]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[120,126,175,176,322,318,384,382,316,320,169,170],"all_points_y":[88,200,206,273,268,199,202,77,77,8,10,84]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_158.jpg20022":{"filename":"IMG_158.jpg","size":20022,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[79,91,144,130],"all_points_y":[84,170,163,78]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[65,66,160,155],"all_points_y":[66,198,196,62]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[19,19,69,66,163,158,203,199,155,158,60,66],"all_points_y":[60,202,194,242,242,192,195,53,63,15,21,71]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_159.jpg23088":{"filename":"IMG_159.jpg","size":23088,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[101,93,126,133],"all_points_y":[133,192,197,136]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[79,77,158,152],"all_points_y":[124,231,230,124]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[48,41,81,76,160,155,187,174,148,152,80,80],"all_points_y":[116,232,228,252,252,226,231,117,126,92,92,126]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_160.jpg14487":{"filename":"IMG_160.jpg","size":14487,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[46,48,92,88],"all_points_y":[118,165,163,113]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[37,41,114,113],"all_points_y":[81,199,191,81]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[42,111,112,143,144,112,112,44,42,3,1,40],"all_points_y":[45,48,85,86,188,190,225,233,194,197,81,82]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_161.jpg35869":{"filename":"IMG_161.jpg","size":35869,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[168,167,282,280],"all_points_y":[112,192,192,108]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[132,131,292,292],"all_points_y":[90,203,205,87]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[79,76,132,122,292,289,349,347,288,293,121,135],"all_points_y":[93,201,200,258,265,201,210,84,91,26,42,95]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_162.jpg42329":{"filename":"IMG_162.jpg","size":42329,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[157,161,280,276],"all_points_y":[115,199,193,105]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[126,131,311,303],"all_points_y":[99,222,221,90]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[63,69,132,113,320,305,341,334,299,303,112,127],"all_points_y":[100,232,219,246,245,216,229,79,94,30,44,103]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_163.jpg24084":{"filename":"IMG_163.jpg","size":24084,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[89,90,134,132],"all_points_y":[110,170,168,108]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[71,73,144,146],"all_points_y":[93,193,196,93]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[73,146,146,185,176,140,142,70,74,38,35,74],"all_points_y":[58,60,97,96,203,195,229,226,190,197,92,94]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_164.jpg46002":{"filename":"IMG_164.jpg","size":46002,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[104,114,242,241],"all_points_y":[79,190,182,74]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[73,101,315,302],"all_points_y":[68,231,198,50]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[37,47,56,72,100,107,161,321,312,355,357,348,299,346,195,43,73],"all_points_y":[62,192,192,264,226,300,297,268,197,220,118,40,54,15,20,38,70]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_165.jpg23943":{"filename":"IMG_165.jpg","size":23943,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[78,80,144,136],"all_points_y":[119,205,200,114]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[74,73,168,160],"all_points_y":[105,233,227,102]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[36,31,75,73,177,164,188,180,158,170,71,74],"all_points_y":[98,237,231,277,270,223,232,90,105,76,78,106]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_166.jpg27222":{"filename":"IMG_166.jpg","size":27222,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[75,142,146,128,72,67],"all_points_y":[124,132,160,165,162,142]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[49,72,196,157],"all_points_y":[69,248,221,62]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[24,41,51,74,70,144,176,177,218,224,224,191,224,223,204,155,175,43,50],"all_points_y":[32,218,263,243,267,257,252,247,234,232,225,216,214,113,54,62,23,27,72]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_167.jpg17439":{"filename":"IMG_167.jpg","size":17439,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[123,117,145,151],"all_points_y":[160,183,188,166]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[40,67,187,157],"all_points_y":[151,252,219,116]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[21,139,156,195,220,183,202,106,80,69,14,1,1,38],"all_points_y":[99,62,118,107,197,218,271,298,298,248,265,216,163,152]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_168.jpg17812":{"filename":"IMG_168.jpg","size":17812,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[108,105,136,134],"all_points_y":[89,157,156,92]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[66,70,167,161],"all_points_y":[67,178,176,63]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[16,20,72,76,162,162,215,211,161,160,65,68],"all_points_y":[67,181,176,210,211,173,176,52,67,13,19,70]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_169.jpg24933":{"filename":"IMG_169.jpg","size":24933,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[116,108,130,137],"all_points_y":[126,188,192,130]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[66,64,162,159],"all_points_y":[92,222,224,96]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[66,160,155,202,208,160,156,156,62,68,64,17,21,70],"all_points_y":[48,50,98,88,226,222,223,271,268,218,218,222,84,95]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_170.jpg40002":{"filename":"IMG_170.jpg","size":40002,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[180,236,255,195],"all_points_y":[208,162,185,230]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[86,77,306,303],"all_points_y":[96,248,262,107]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[58,40,40,79,44,180,334,301,347,354,344,300,306,90,89],"all_points_y":[75,204,265,248,290,299,299,258,281,162,92,112,36,23,96]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_171.jpg17521":{"filename":"IMG_171.jpg","size":17521,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[93,98,133,128],"all_points_y":[106,158,155,103]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[30,38,163,160],"all_points_y":[92,203,199,86]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[25,163,159,216,216,162,164,41,39,5,1,32],"all_points_y":[32,27,89,90,193,198,251,256,199,194,100,94]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_172.jpg30798":{"filename":"IMG_172.jpg","size":30798,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[202,178,207,255],"all_points_y":[96,157,180,140]},"region_attributes":{"name":"item_sq"}},{"shape_attributes":{"name":"polygon","all_points_x":[144,154,290,278],"all_points_y":[87,208,196,76]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[82,95,155,160,294,284,329,314,275,284,136,144],"all_points_y":[92,209,207,270,254,192,182,82,81,19,29,90]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_173.jpg23233":{"filename":"IMG_173.jpg","size":23233,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[88,91,127,124],"all_points_y":[103,160,159,101]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[66,72,165,164],"all_points_y":[70,204,201,69]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[20,30,74,68,168,162,181,184,162,174,57,70],"all_points_y":[68,214,203,251,249,200,218,61,72,44,51,73]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_174.jpg28656":{"filename":"IMG_174.jpg","size":28656,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[76,108,107,74],"all_points_y":[130,130,145,143]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[12,94,205,109],"all_points_y":[109,269,194,65]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[112,109,150,224,224,202,223,224,181,105,95,81,35,8,0,1,16,0,1],"all_points_y":[26,70,48,132,178,194,188,198,236,282,270,284,207,154,132,99,112,82,64]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_175.jpg18443":{"filename":"IMG_175.jpg","size":18443,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[68,61,175,166],"all_points_y":[88,233,236,90]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[14,2,63,51,184,172,222,224,218,164,175,63,71],"all_points_y":[85,232,231,286,290,235,236,146,89,94,52,50,89]},"region_attributes":{"name":"outerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[92,89,87,91,99,108,116,129,139,142,144,143,138,132,124,108,97],"all_points_y":[145,161,175,183,198,207,209,208,201,189,178,162,150,142,136,134,138]},"region_attributes":{"name":"item_circ"}}],"file_attributes":{}},"IMG_176.jpg24093":{"filename":"IMG_176.jpg","size":24093,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[95,92,112,109],"all_points_y":[118,195,193,118]},"region_attributes":{"name":"item_rect_slim"}},{"shape_attributes":{"name":"polygon","all_points_x":[73,76,162,159],"all_points_y":[98,219,217,95]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[36,35,78,72,164,159,174,171,155,163,68,74],"all_points_y":[99,223,217,259,256,213,225,86,99,73,78,101]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_177.jpg17580":{"filename":"IMG_177.jpg","size":17580,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[113,116,159,156],"all_points_y":[110,163,162,110]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[81,86,163,157],"all_points_y":[109,219,214,105]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[48,50,88,84,166,160,197,193,155,157,79,82],"all_points_y":[109,225,217,255,252,213,217,104,107,70,73,111]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_178.jpg22572":{"filename":"IMG_178.jpg","size":22572,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[89,88,128,127],"all_points_y":[133,189,189,134]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[72,70,160,162],"all_points_y":[101,234,232,103]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[29,26,70,65,166,159,204,204,160,168,69,73],"all_points_y":[100,240,231,280,275,229,241,102,106,64,65,105]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_179.jpg14341":{"filename":"IMG_179.jpg","size":14341,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[52,54,91,88],"all_points_y":[126,154,152,125]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[43,45,118,117],"all_points_y":[81,197,194,82]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[4,43,44,113,116,144,144,116,115,47,45,6],"all_points_y":[78,80,42,48,86,88,188,191,227,234,195,199]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_180.jpg23144":{"filename":"IMG_180.jpg","size":23144,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[104,104,134,133],"all_points_y":[132,173,174,134]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[81,79,156,161],"all_points_y":[95,208,210,99]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[47,44,83,73,157,152,156,187,190,159,165,79,82],"all_points_y":[92,214,204,234,238,207,205,221,97,102,63,60,98]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}},"IMG_181.jpg14152":{"filename":"IMG_181.jpg","size":14152,"regions":[{"shape_attributes":{"name":"polygon","all_points_x":[64,66,97,95],"all_points_y":[149,188,185,144]},"region_attributes":{"name":"item_rect"}},{"shape_attributes":{"name":"polygon","all_points_x":[35,38,109,106],"all_points_y":[90,202,193,90]},"region_attributes":{"name":"innerbox"}},{"shape_attributes":{"name":"polygon","all_points_x":[38,107,105,136,140,105,108,43,41,38,5,2,38],"all_points_y":[49,55,92,91,193,192,228,236,201,199,209,82,90]},"region_attributes":{"name":"outerbox"}}],"file_attributes":{}}},"_via_attributes":{"region":{"name":{"type":"dropdown","description":"","options":{"outerbox":"","innerbox":"","item_sq":"","item_rect":"","item_rect_slim":"","item_circ":""},"default_options":{}}},"file":{}}} \ No newline at end of file diff --git a/mask_rcnn/dataset/val_img_results/Figure_1.png b/mask_rcnn/dataset/val_img_results/Figure_1.png new file mode 100644 index 00000000..ec7ae97f Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_1.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_10.png b/mask_rcnn/dataset/val_img_results/Figure_10.png new file mode 100644 index 00000000..eac33a4d Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_10.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_11.png b/mask_rcnn/dataset/val_img_results/Figure_11.png new file mode 100644 index 00000000..dd77bdae Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_11.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_12.png b/mask_rcnn/dataset/val_img_results/Figure_12.png new file mode 100644 index 00000000..02e8d818 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_12.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_13.png b/mask_rcnn/dataset/val_img_results/Figure_13.png new file mode 100644 index 00000000..f073487a Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_13.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_14.png b/mask_rcnn/dataset/val_img_results/Figure_14.png new file mode 100644 index 00000000..934d874a Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_14.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_15.png b/mask_rcnn/dataset/val_img_results/Figure_15.png new file mode 100644 index 00000000..733dc30f Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_15.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_16.png b/mask_rcnn/dataset/val_img_results/Figure_16.png new file mode 100644 index 00000000..6e9b4961 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_16.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_17.png b/mask_rcnn/dataset/val_img_results/Figure_17.png new file mode 100644 index 00000000..88f1028b Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_17.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_18.png b/mask_rcnn/dataset/val_img_results/Figure_18.png new file mode 100644 index 00000000..e24b821c Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_18.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_19.png b/mask_rcnn/dataset/val_img_results/Figure_19.png new file mode 100644 index 00000000..34adaec8 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_19.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_2.png b/mask_rcnn/dataset/val_img_results/Figure_2.png new file mode 100644 index 00000000..995faede Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_2.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_20.png b/mask_rcnn/dataset/val_img_results/Figure_20.png new file mode 100644 index 00000000..76946653 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_20.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_21.png b/mask_rcnn/dataset/val_img_results/Figure_21.png new file mode 100644 index 00000000..90b8dd19 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_21.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_22.png b/mask_rcnn/dataset/val_img_results/Figure_22.png new file mode 100644 index 00000000..3d48ba55 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_22.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_23.png b/mask_rcnn/dataset/val_img_results/Figure_23.png new file mode 100644 index 00000000..0b175fdb Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_23.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_24.png b/mask_rcnn/dataset/val_img_results/Figure_24.png new file mode 100644 index 00000000..25070069 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_24.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_25.png b/mask_rcnn/dataset/val_img_results/Figure_25.png new file mode 100644 index 00000000..a05a9aae Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_25.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_26.png b/mask_rcnn/dataset/val_img_results/Figure_26.png new file mode 100644 index 00000000..ac1ab512 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_26.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_27.png b/mask_rcnn/dataset/val_img_results/Figure_27.png new file mode 100644 index 00000000..721770fb Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_27.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_28.png b/mask_rcnn/dataset/val_img_results/Figure_28.png new file mode 100644 index 00000000..94dfe667 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_28.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_29.png b/mask_rcnn/dataset/val_img_results/Figure_29.png new file mode 100644 index 00000000..109a5cf9 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_29.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_3.png b/mask_rcnn/dataset/val_img_results/Figure_3.png new file mode 100644 index 00000000..d761ce2e Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_3.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_30.png b/mask_rcnn/dataset/val_img_results/Figure_30.png new file mode 100644 index 00000000..75186b3c Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_30.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_4.png b/mask_rcnn/dataset/val_img_results/Figure_4.png new file mode 100644 index 00000000..f2ca7041 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_4.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_5.png b/mask_rcnn/dataset/val_img_results/Figure_5.png new file mode 100644 index 00000000..dcaad0f6 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_5.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_6.png b/mask_rcnn/dataset/val_img_results/Figure_6.png new file mode 100644 index 00000000..a0bb395f Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_6.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_7.png b/mask_rcnn/dataset/val_img_results/Figure_7.png new file mode 100644 index 00000000..feb4ce38 Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_7.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_8.png b/mask_rcnn/dataset/val_img_results/Figure_8.png new file mode 100644 index 00000000..9ff3b2bb Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_8.png differ diff --git a/mask_rcnn/dataset/val_img_results/Figure_9.png b/mask_rcnn/dataset/val_img_results/Figure_9.png new file mode 100644 index 00000000..2fc94c9a Binary files /dev/null and b/mask_rcnn/dataset/val_img_results/Figure_9.png differ diff --git a/mask_rcnn/dataset/via.html b/mask_rcnn/dataset/via.html new file mode 100644 index 00000000..8f538b06 --- /dev/null +++ b/mask_rcnn/dataset/via.html @@ -0,0 +1,10946 @@ + + + + + + VGG Image Annotator + + + + + + + + + + + + + + + + + + Rectangular region shape + + + + Circular region shape + + + + Elliptical region shape + + + + Polygon region shape + + + + Point region shape + + + + Polyline region shape + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + n + + + + x + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+ + +
+
+ Region Annotations + File Annotations + + × + ↑ + ↓ + + + − +
+ +
+ +
+ +
+ +
+ + +
+ Open Project + Save Project + Update Project Settings + + + Import Annotations from CSV + Download Annotations as CSV + + Switch to Image Grid View + Toggle Annotation Editor + + Previous + Toggle Filename List + Next + + Zoom In + Zoom Out + + Select All Regions + Copy Regions + Paste Regions + Paste Region in Multiple Images + Undo Regions Pasted in Multiple Images + Delete Region +
+ + +
+ + +
+ + + + + + +
+
+ + + + Sorry, your browser does not support HTML5 Canvas functionality which is required for this application. + +
+
+ +
+ Group by  +
+ +
+ Selected + 0 + of + 0 + images in current group, show + + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+

Settings

+
+
+
Project Name
+
+ +
+ +
+
+ +
+
+
Default Path
+
If all images in your project are saved in a single folder, set the default path to the location of this folder. The VIA application will load images from this folder by default. Note: a default path of "./" indicates that the folder containing via.html application file also contains the images in this project. For example: /datasets/VOC2012/JPEGImages/ or C:\Documents\data\ (note the trailing / and \)
+
+ +
+ +
+
+ +
+
+
Search Path List
+
If you define multiple paths, all these folders will be searched to find images in this project. We do not recommend this approach as it is computationally expensive to search for images in multiple folders.
    +
    + +
    + +
    +
    + +
    +
    +
    Region Label
    +
    By default, each region in an image is labelled using the region-id. Here, you can select a more descriptive labelling of regions.
    +
    + +
    + +
    +
    + +
    +
    +
    Region Colour
    +
    By default, each region is drawn using a single colour. Using this setting, you can assign a unique colour to regions grouped according to a region attribute.
    +
    + +
    + +
    +
    + +
    +
    +
    Region Label Font
    +
    Font size and font family for showing region labels.
    +
    + +
    + +
    +
    + +
    +
    +
    Preload Buffer Size
    +
    Images are preloaded in buffer to allow smoother navigation of next/prev images. A large buffer size may slow down the overall browser performance. To disable preloading, set buffer size to 0.
    +
    +
    + +
    +
    + +
    +
    +
    On-image Annotation Editor
    +
    When a single region is selected, the on-image annotation editor is gets activated which the user to update annotations of this region. By default, this on-image annotation editor is placed near the selected region.
    +
    + +
    + +
    +
    + +
    + + +
    +
    + +
    +

    File Not Found

    +

    Filename:

    + +

    We recommend that you update the default path in project settings to the folder which contains this image.

    + +

    A temporary fix is to use browser's file selector to manually locate and add this file. We do not recommend this approach because it requires you to repeat this process every time your load this project in the VIA application.

    +
    + +
    +
      +
    • To start annotation, select images (or, add images from URL or absolute path) and draw regions
      • +
    • Use attribute editor to define attributes (e.g. name) and annotation editor to describe each region (e.g. cat) using these attributes.
      • +
    • Remember to save your project before closing this application so that you can load it later to continue annotation.
      • +
    • For help, see the Getting Started page and pre-loaded demo: image annotation and face annotation.
      • +
    + +
    + +
    +

    A more detailed user guide (with screenshots and descriptions) is available here.

    +
      +
    1. Load Images: The first step is to load all the images that you wish to annotate. There are multiple ways to add images to a VIA project. Choose the method that suits your use case. +
        +
      • Method 1: Selecting local files using browser's file selector +
          +
        1. Click Project → Add local files
          2. +
        2. Select desired images and click Open
          3. +
        +
        • +
      • Method 2: Adding files from URL or absolute path +
          +
        1. Click Project → Add files from URL
          2. +
        2. Enter URL and click OK
          3. +
        +
        • +
      • Method 3: Adding files from list of url or absolute path stored in text file +
          +
        1. Create a text file containing URL and absolute path (one per line)
          2. +
        2. Click Project → Add url or path from text file
          3. +
        3. Select the text file and click Open
          4. +
        +
        • +
      +
      2. +
    2. Draw Regions: Select a region shape (rectangle, circle, ellipse, polygon, point, polyline) from the left sidebar and draw regions as follows: + +
        +
      • Rectangle, Circle and Ellipse +
          +
        • Press left mouse button, drag mouse cursor and release mouse button.
          • +
        • To define a point inside an existing region, click inside the region to select it (if not already selected), now press left mouse button, drag and release to draw region inside existing region.
          • +
        • To select, click inside the region. If the click point contains multiple regions, then clicking multiple times at that location shuffles selection through those regions.
          • +
        +
        • +
      + +
        +
      • Point +
          +
        • Click to define points.
          • +
        • To draw a region inside existing region, click inside the region to select it (if not already selected), now click again to define the point.
          • +
        • To select, click on (or near) the existing point.
          • +
        +
        • +
      + +
        +
      • Polygon and Polyline +
          +
        • Click to define vertices.
          • +
        • Press [Enter] to finish drawing the region or press [Esc] to cancel.
          • +
        • If the first vertex needs to be defined inside an existing region, click inside the region to select it (if not already selected), now click again to define the vertex.
          • +
        • To select, click inside the region. If the click point contains multiple regions, then clicking multiple times at that location shuffles selection through those regions.
          • +
        +
        • +
      +
      3. + +
    3. Create Annotations: For a more detailed description of this step, see Creating Annotations : VIA User Guide. Click the View → Toggle attributes editor to show attributes editor panel in left sidebar and add the desired file or region attributes (e.g. name). Now click View → Toggle annotations editor to show the annotation editor panel in the bottom side. Update the annotations for each region.
      4. +
    4. Export Annotations: To export the annotations in json or csv format, click Annotation → Export annotations in top menubar.
      5. +
    5. Save Project: To save the project, click Project → Save in top menubar.
      6. +
    +
    + +
    +
    + + + + + +
    Loading ...
    +
    +
    + +
    +
    + + + + + +
    + +

    Version 2.0.9

    +

    VGG Image Annotator (VIA) is an image annotation tool that can be used to define regions in an image and create textual descriptions of those regions. VIA is an open source project developed at the Visual Geometry Group and released under the BSD-2 clause license.

    +

    Here is a list of some salient features of VIA: +

      +
    • based solely on HTML, CSS and Javascript (no external javascript libraries)
      • +
    • can be used off-line (full application in a single html file of size < 400KB)
      • +
    • requires nothing more than a modern web browser (tested on Firefox, Chrome and Safari)
      • +
    • supported region shapes: rectangle, circle, ellipse, polygon, point and polyline
      • +
    • import/export of region data in csv and json file format
      • +
    +

    +

    For more details, visit http://www.robots.ox.ac.uk/~vgg/software/via/.

    +

     

    +

    Copyright © 2016-2019, Abhishek Dutta,Visual Geometry Group, Oxford University and VIA Contributors.

    +
    + +
    + 
    +Copyright (c) 2016-2019, Abhishek Dutta, Visual Geometry Group, Oxford University and VIA Contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
    +
    +
    +
    + + +
    + + + + + + + diff --git a/mask_rcnn/eval_on_val_set.py b/mask_rcnn/eval_on_val_set.py new file mode 100644 index 00000000..2d4b6141 --- /dev/null +++ b/mask_rcnn/eval_on_val_set.py @@ -0,0 +1,266 @@ +from mrcnn.model import log +import mrcnn.model as modellib +from mrcnn.visualize import display_images +import mrcnn.visualize as visualize +import mrcnn.utils as utils +from mrcnn.config import Config +import sys +import random +import math +import re +import time +import numpy as np +import tensorflow as tf +import matplotlib +import matplotlib.pyplot as plt +import matplotlib.patches as patches +import os +import sys +import json +import datetime +import skimage.draw +import cv2 + +# TODO: update this path +PP_WEIGHTS_PATH = "models/mask_rcnn_pointless_package_0050.h5" + + +class PPConfig(Config): + """Configuration for training on the toy dataset. + Derives from the base Config class and overrides some values. + """ + # Give the configuration a recognizable name + NAME = "pointless_package" + + # We use a GPU with 12GB memory, which can fit two images. + # Adjust down if you use a smaller GPU. + IMAGES_PER_GPU = 1 + + # Number of classes (including background) + # Background + outerbox + innerbox + item_rect + item_rect_slim + item_sq + item_circ + NUM_CLASSES = 1 + 6 + + # Number of training steps per epoch + STEPS_PER_EPOCH = 100 + + # Skip detections with < 90% confidence + DETECTION_MIN_CONFIDENCE = 0.75 + +class PPDataset(utils.Dataset): + + def load_dataset(self, dataset_dir, subset): + """Load a subset of the Balloon dataset. + dataset_dir: Root directory of the dataset. + subset: Subset to load: train or val + """ + # Add classes. We have only one class to add. + self.add_class("pointless_package", 1, "outerbox") + self.add_class("pointless_package", 2, "innerbox") + self.add_class("pointless_package", 3, "item_sq") + self.add_class("pointless_package", 4, "item_rect") + self.add_class("pointless_package", 5, "item_rect_slim") + self.add_class("pointless_package", 6, "item_circ") + + # Train or validation dataset? + assert subset in ["train", "val"] + dataset_dir = os.path.join(dataset_dir, subset) + + # Load annotations + # VGG Image Annotator (up to version 1.6) saves each image in the form: + # { 'filename': '28503151_5b5b7ec140_b.jpg', + # 'regions': { + # '0': { + # 'region_attributes': {}, + # 'shape_attributes': { + # 'all_points_x': [...], + # 'all_points_y': [...], + # 'name': 'polygon'}}, + # ... more regions ... + # }, + # 'size': 100202 + # } + # We mostly care about the x and y coordinates of each region + # Note: In VIA 2.0, regions was changed from a dict to a list. + annotations = json.load( + open(os.path.join(dataset_dir, "via_region_data.json"))) + annotations = list(annotations.values()) # don't need the dict keys + + # The VIA tool saves images in the JSON even if they don't have any + # annotations. Skip unannotated images. + annotations = [a for a in annotations if a['regions']] + + # Add images + for a in annotations: + # Get the x, y coordinaets of points of the polygons that make up + # the outline of each object instance. These are stores in the + # shape_attributes (see json format above) + # The if condition is needed to support VIA versions 1.x and 2.x. + if type(a['regions']) is dict: + polygons = [r['shape_attributes'] + for r in a['regions'].values()] + else: + polygons = [r['shape_attributes'] for r in a['regions']] + + # load_mask() needs the image size to convert polygons to masks. + # Unfortunately, VIA doesn't include it in JSON, so we must read + # the image. This is only managable since the dataset is tiny. + image_path = os.path.join(dataset_dir, a['filename']) + image = skimage.io.imread(image_path) + height, width = image.shape[:2] + + class_list = [r['region_attributes'] for r in a['regions']] + + self.add_image( + "pointless_package", + image_id=a['filename'], # use file name as a unique image id + path=image_path, + width=width, height=height, + class_list=class_list, + polygons=polygons) + + def load_mask(self, image_id): + """Generate instance masks for an image. + Returns: + masks: A bool array of shape [height, width, instance count] with + one mask per instance. + class_ids: a 1D array of class IDs of the instance masks. + """ + class_ids = list() + # If not a pointless_package dataset image, delegate to parent class. + image_info = self.image_info[image_id] + # if image_info["source"] != "pointless_package": + # return super(self.__class__, self).load_mask(image_id) + + # Convert polygons to a bitmap mask of shape + # [height, width, instance_count] + info = self.image_info[image_id] + # print("\n\n\nIMAGE INFO:", info, "\n\n\n\n") + + for box_type in info['class_list']: + # print(box_type['name']) + class_ids.append(self.class_names.index(str(box_type['name']))) + # print(class_ids) + # print(self.class_names) + + mask = np.zeros([info["height"], info["width"], len(info["polygons"])], + dtype=np.uint8) + for i, p in enumerate(info["polygons"]): + # Get indexes of pixels inside the polygon and set them to 1 + rr, cc = skimage.draw.polygon(p['all_points_y'], p['all_points_x']) + mask[rr, cc, i] = 1 + # Return mask, and array of class IDs of each instance. Since we have + # one class ID only, we return an array of 1s + return mask.astype(np.bool), np.asarray(class_ids, dtype=np.int32) + + def image_reference(self, image_id): + """Return the path of the image.""" + info = self.image_info[image_id] + if info["source"] == "pointless_package": + return info["path"] + else: + super(self.__class__, self).image_reference(image_id) + + +config = PPConfig() +ROOT_DIR = os.getcwd() +PP_DIR = os.path.join(ROOT_DIR, "dataset/") + +# Override the training configurations with a few +# changes for inferencing. +class InferenceConfig(config.__class__): + # Run detection on one image at a time + GPU_COUNT = 1 + IMAGES_PER_GPU = 1 + +config = InferenceConfig() +config.display() + +# Device to load the neural network on. +# Useful if you're training a model on the same +# machine, in which case use CPU and leave the +# GPU for training. +DEVICE = "/cpu:0" # /cpu:0 or /gpu:0 + +# Inspect the model in training or inference modes +# values: 'inference' or 'training' +# TODO: code for 'training' test mode not ready yet +TEST_MODE = "inference" + + +def get_ax(rows=1, cols=1, size=16): + """Return a Matplotlib Axes array to be used in + all visualizations in the notebook. Provide a + central point to control graph sizes. + + Adjust the size attribute to control how big to render images + """ + _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows)) + return ax + + +MY_ABS_PATH = "./" +my_model_dir = MY_ABS_PATH + 'models/' + +# Load validation dataset +dataset = PPDataset() +dataset.load_dataset(PP_DIR, "val") + +# Must call before using the dataset +dataset.prepare() + +print("Images: {}\nClasses: {}".format( + len(dataset.image_ids), dataset.class_names)) + +# Create model in inference mode +with tf.device(DEVICE): + model = modellib.MaskRCNN(mode="inference", model_dir=my_model_dir, + config=config) + +# Set path to balloon weights file + +# Download file from the Releases page and set its path +# https://github.com/matterport/Mask_RCNN/releases +# weights_path = "/path/to/mask_rcnn_balloon.h5" + +# Or, load the last model you trained +# weights_path = model.find_last()[1] +weights_path = PP_WEIGHTS_PATH + +# Load weights +print("Loading weights ", weights_path) +model.load_weights(weights_path, by_name=True) + +# print(dataset.image_ids) +# image_id = random.choice(dataset.image_ids) +for number in range(150, 181): + # image_id = number + # image, image_meta, gt_class_id, gt_bbox, gt_mask =\ + # modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) + # info = dataset.image_info[image_id] + # print("image ID: {}.{} ({}) {}".format(info["source"], info["id"], image_id, + # dataset.image_reference(image_id))) + + image = cv2.imread('./dataset/val/IMG_'+str(number)+'.jpg') + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + + # Run object detection + results = model.detect([image], verbose=1) + + # Display results + r = results[0] + visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], + dataset.class_names, r['scores'], + title="Predictions") + + N = r['rois'].shape[0] + class_ids = r['class_ids'] + masks = r['masks'] + + class_names = np.asarray(dataset.class_names) + + print(class_names[class_ids]) + + # score_card = [masks[:, :, i].sum() for i in range(N)] + score_card2 = masks.sum(axis=0).sum(axis=0) + # print(score_card) + print(score_card2) diff --git a/mask_rcnn/models/.gitkeep b/mask_rcnn/models/.gitkeep new file mode 100644 index 00000000..74c20143 --- /dev/null +++ b/mask_rcnn/models/.gitkeep @@ -0,0 +1 @@ +DUMMY \ No newline at end of file diff --git a/mask_rcnn/models/serving_model/.gitkeep b/mask_rcnn/models/serving_model/.gitkeep new file mode 100644 index 00000000..74c20143 --- /dev/null +++ b/mask_rcnn/models/serving_model/.gitkeep @@ -0,0 +1 @@ +DUMMY \ No newline at end of file diff --git a/mask_rcnn/mrcnn/__init__.py b/mask_rcnn/mrcnn/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/mask_rcnn/mrcnn/__init__.py @@ -0,0 +1 @@ + diff --git a/mask_rcnn/mrcnn/config.py b/mask_rcnn/mrcnn/config.py new file mode 100644 index 00000000..5bffb33d --- /dev/null +++ b/mask_rcnn/mrcnn/config.py @@ -0,0 +1,236 @@ +""" +Mask R-CNN +Base Configurations class. + +Copyright (c) 2017 Matterport, Inc. +Licensed under the MIT License (see LICENSE for details) +Written by Waleed Abdulla +""" + +import numpy as np + + +# Base Configuration Class +# Don't use this class directly. Instead, sub-class it and override +# the configurations you need to change. + +class Config(object): + """Base configuration class. For custom configurations, create a + sub-class that inherits from this one and override properties + that need to be changed. + """ + # Name the configurations. For example, 'COCO', 'Experiment 3', ...etc. + # Useful if your code needs to do things differently depending on which + # experiment is running. + NAME = None # Override in sub-classes + + # NUMBER OF GPUs to use. When using only a CPU, this needs to be set to 1. + GPU_COUNT = 1 + + # Number of images to train with on each GPU. A 12GB GPU can typically + # handle 2 images of 1024x1024px. + # Adjust based on your GPU memory and image sizes. Use the highest + # number that your GPU can handle for best performance. + IMAGES_PER_GPU = 2 + + # Number of training steps per epoch + # This doesn't need to match the size of the training set. Tensorboard + # updates are saved at the end of each epoch, so setting this to a + # smaller number means getting more frequent TensorBoard updates. + # Validation stats are also calculated at each epoch end and they + # might take a while, so don't set this too small to avoid spending + # a lot of time on validation stats. + STEPS_PER_EPOCH = 1000 + + # Number of validation steps to run at the end of every training epoch. + # A bigger number improves accuracy of validation stats, but slows + # down the training. + VALIDATION_STEPS = 50 + + # Backbone network architecture + # Supported values are: resnet50, resnet101. + # You can also provide a callable that should have the signature + # of model.resnet_graph. If you do so, you need to supply a callable + # to COMPUTE_BACKBONE_SHAPE as well + BACKBONE = "resnet101" + + # Only useful if you supply a callable to BACKBONE. Should compute + # the shape of each layer of the FPN Pyramid. + # See model.compute_backbone_shapes + COMPUTE_BACKBONE_SHAPE = None + + # The strides of each layer of the FPN Pyramid. These values + # are based on a Resnet101 backbone. + BACKBONE_STRIDES = [4, 8, 16, 32, 64] + + # Size of the fully-connected layers in the classification graph + FPN_CLASSIF_FC_LAYERS_SIZE = 1024 + + # Size of the top-down layers used to build the feature pyramid + TOP_DOWN_PYRAMID_SIZE = 256 + + # Number of classification classes (including background) + NUM_CLASSES = 1 # Override in sub-classes + + # Length of square anchor side in pixels + RPN_ANCHOR_SCALES = (32, 64, 128, 256, 512) + + # Ratios of anchors at each cell (width/height) + # A value of 1 represents a square anchor, and 0.5 is a wide anchor + RPN_ANCHOR_RATIOS = [0.5, 1, 2] + + # Anchor stride + # If 1 then anchors are created for each cell in the backbone feature map. + # If 2, then anchors are created for every other cell, and so on. + RPN_ANCHOR_STRIDE = 1 + + # Non-max suppression threshold to filter RPN proposals. + # You can increase this during training to generate more propsals. + RPN_NMS_THRESHOLD = 0.7 + + # How many anchors per image to use for RPN training + RPN_TRAIN_ANCHORS_PER_IMAGE = 256 + + # ROIs kept after tf.nn.top_k and before non-maximum suppression + PRE_NMS_LIMIT = 6000 + + # ROIs kept after non-maximum suppression (training and inference) + POST_NMS_ROIS_TRAINING = 2000 + POST_NMS_ROIS_INFERENCE = 1000 + + # If enabled, resizes instance masks to a smaller size to reduce + # memory load. Recommended when using high-resolution images. + USE_MINI_MASK = False + MINI_MASK_SHAPE = (56, 56) # (height, width) of the mini-mask + + # Input image resizing + # Generally, use the "square" resizing mode for training and predicting + # and it should work well in most cases. In this mode, images are scaled + # up such that the small side is = IMAGE_MIN_DIM, but ensuring that the + # scaling doesn't make the long side > IMAGE_MAX_DIM. Then the image is + # padded with zeros to make it a square so multiple images can be put + # in one batch. + # Available resizing modes: + # none: No resizing or padding. Return the image unchanged. + # square: Resize and pad with zeros to get a square image + # of size [max_dim, max_dim]. + # pad64: Pads width and height with zeros to make them multiples of 64. + # If IMAGE_MIN_DIM or IMAGE_MIN_SCALE are not None, then it scales + # up before padding. IMAGE_MAX_DIM is ignored in this mode. + # The multiple of 64 is needed to ensure smooth scaling of feature + # maps up and down the 6 levels of the FPN pyramid (2**6=64). + # crop: Picks random crops from the image. First, scales the image based + # on IMAGE_MIN_DIM and IMAGE_MIN_SCALE, then picks a random crop of + # size IMAGE_MIN_DIM x IMAGE_MIN_DIM. Can be used in training only. + # IMAGE_MAX_DIM is not used in this mode. + IMAGE_RESIZE_MODE = "square" + IMAGE_MIN_DIM = 800 + IMAGE_MAX_DIM = 1024 + # Minimum scaling ratio. Checked after MIN_IMAGE_DIM and can force further + # up scaling. For example, if set to 2 then images are scaled up to double + # the width and height, or more, even if MIN_IMAGE_DIM doesn't require it. + # However, in 'square' mode, it can be overruled by IMAGE_MAX_DIM. + IMAGE_MIN_SCALE = 0 + # Number of color channels per image. RGB = 3, grayscale = 1, RGB-D = 4 + # Changing this requires other changes in the code. See the WIKI for more + # details: https://github.com/matterport/Mask_RCNN/wiki + IMAGE_CHANNEL_COUNT = 3 + + # Image mean (RGB) + MEAN_PIXEL = np.array([123.7, 116.8, 103.9]) + + # Number of ROIs per image to feed to classifier/mask heads + # The Mask RCNN paper uses 512 but often the RPN doesn't generate + # enough positive proposals to fill this and keep a positive:negative + # ratio of 1:3. You can increase the number of proposals by adjusting + # the RPN NMS threshold. + TRAIN_ROIS_PER_IMAGE = 200 + + # Percent of positive ROIs used to train classifier/mask heads + ROI_POSITIVE_RATIO = 0.33 + + # Pooled ROIs + POOL_SIZE = 7 + MASK_POOL_SIZE = 14 + + # Shape of output mask + # To change this you also need to change the neural network mask branch + MASK_SHAPE = [28, 28] + + # Maximum number of ground truth instances to use in one image + MAX_GT_INSTANCES = 100 + + # Bounding box refinement standard deviation for RPN and final detections. + RPN_BBOX_STD_DEV = np.array([0.1, 0.1, 0.2, 0.2]) + BBOX_STD_DEV = np.array([0.1, 0.1, 0.2, 0.2]) + + # Max number of final detections + DETECTION_MAX_INSTANCES = 100 + + # Minimum probability value to accept a detected instance + # ROIs below this threshold are skipped + DETECTION_MIN_CONFIDENCE = 0.7 + + # Non-maximum suppression threshold for detection + DETECTION_NMS_THRESHOLD = 0.3 + + # Learning rate and momentum + # The Mask RCNN paper uses lr=0.02, but on TensorFlow it causes + # weights to explode. Likely due to differences in optimizer + # implementation. + LEARNING_RATE = 0.001 + LEARNING_MOMENTUM = 0.9 + + # Weight decay regularization + WEIGHT_DECAY = 0.0001 + + # Loss weights for more precise optimization. + # Can be used for R-CNN training setup. + LOSS_WEIGHTS = { + "rpn_class_loss": 1., + "rpn_bbox_loss": 1., + "mrcnn_class_loss": 1., + "mrcnn_bbox_loss": 1., + "mrcnn_mask_loss": 1. + } + + # Use RPN ROIs or externally generated ROIs for training + # Keep this True for most situations. Set to False if you want to train + # the head branches on ROI generated by code rather than the ROIs from + # the RPN. For example, to debug the classifier head without having to + # train the RPN. + USE_RPN_ROIS = True + + # Train or freeze batch normalization layers + # None: Train BN layers. This is the normal mode + # False: Freeze BN layers. Good when using a small batch size + # True: (don't use). Set layer in training mode even when predicting + TRAIN_BN = False # Defaulting to False since batch size is often small + + # Gradient norm clipping + GRADIENT_CLIP_NORM = 5.0 + + def __init__(self): + """Set values of computed attributes.""" + # Effective batch size + self.BATCH_SIZE = self.IMAGES_PER_GPU * self.GPU_COUNT + + # Input image size + if self.IMAGE_RESIZE_MODE == "crop": + self.IMAGE_SHAPE = np.array([self.IMAGE_MIN_DIM, self.IMAGE_MIN_DIM, + self.IMAGE_CHANNEL_COUNT]) + else: + self.IMAGE_SHAPE = np.array([self.IMAGE_MAX_DIM, self.IMAGE_MAX_DIM, + self.IMAGE_CHANNEL_COUNT]) + + # Image meta data length + # See compose_image_meta() for details + self.IMAGE_META_SIZE = 1 + 3 + 3 + 4 + 1 + self.NUM_CLASSES + + def display(self): + """Display Configuration values.""" + print("\nConfigurations:") + for a in dir(self): + if not a.startswith("__") and not callable(getattr(self, a)): + print("{:30} {}".format(a, getattr(self, a))) + print("\n") diff --git a/mask_rcnn/mrcnn/model.py b/mask_rcnn/mrcnn/model.py new file mode 100644 index 00000000..f2200881 --- /dev/null +++ b/mask_rcnn/mrcnn/model.py @@ -0,0 +1,2871 @@ +""" +Mask R-CNN +The main Mask R-CNN model implementation. + +Copyright (c) 2017 Matterport, Inc. +Licensed under the MIT License (see LICENSE for details) +Written by Waleed Abdulla +""" + +import os +import random +import datetime +import re +import math +import logging +from collections import OrderedDict +import multiprocessing +import numpy as np +import warnings +warnings.filterwarnings('ignore', category=DeprecationWarning) +warnings.filterwarnings('ignore', category=FutureWarning) +import tensorflow as tf +import keras +import keras.backend as K +import keras.layers as KL +import keras.engine as KE +import keras.models as KM + +from mrcnn import utils + +# Requires TensorFlow 1.3+ and Keras 2.0.8+. +from distutils.version import LooseVersion +assert LooseVersion(tf.__version__) >= LooseVersion("1.3") +assert LooseVersion(keras.__version__) >= LooseVersion('2.0.8') + + +############################################################ +# Utility Functions +############################################################ + +def log(text, array=None): + """Prints a text message. And, optionally, if a Numpy array is provided it + prints it's shape, min, and max values. + """ + if array is not None: + text = text.ljust(25) + text += ("shape: {:20} ".format(str(array.shape))) + if array.size: + text += ("min: {:10.5f} max: {:10.5f}".format(array.min(),array.max())) + else: + text += ("min: {:10} max: {:10}".format("","")) + text += " {}".format(array.dtype) + print(text) + + +class BatchNorm(KL.BatchNormalization): + """Extends the Keras BatchNormalization class to allow a central place + to make changes if needed. + + Batch normalization has a negative effect on training if batches are small + so this layer is often frozen (via setting in Config class) and functions + as linear layer. + """ + def call(self, inputs, training=None): + """ + Note about training values: + None: Train BN layers. This is the normal mode + False: Freeze BN layers. Good when batch size is small + True: (don't use). Set layer in training mode even when making inferences + """ + return super(self.__class__, self).call(inputs, training=training) + + +def compute_backbone_shapes(config, image_shape): + """Computes the width and height of each stage of the backbone network. + + Returns: + [N, (height, width)]. Where N is the number of stages + """ + if callable(config.BACKBONE): + return config.COMPUTE_BACKBONE_SHAPE(image_shape) + + # Currently supports ResNet only + assert config.BACKBONE in ["resnet50", "resnet101"] + return np.array( + [[int(math.ceil(image_shape[0] / stride)), + int(math.ceil(image_shape[1] / stride))] + for stride in config.BACKBONE_STRIDES]) + + +############################################################ +# Resnet Graph +############################################################ + +# Code adopted from: +# https://github.com/fchollet/deep-learning-models/blob/master/resnet50.py + +def identity_block(input_tensor, kernel_size, filters, stage, block, + use_bias=True, train_bn=True): + """The identity_block is the block that has no conv layer at shortcut + # Arguments + input_tensor: input tensor + kernel_size: default 3, the kernel size of middle conv layer at main path + filters: list of integers, the nb_filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + use_bias: Boolean. To use or not use a bias in conv layers. + train_bn: Boolean. Train or freeze Batch Norm layers + """ + nb_filter1, nb_filter2, nb_filter3 = filters + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = KL.Conv2D(nb_filter1, (1, 1), name=conv_name_base + '2a', + use_bias=use_bias)(input_tensor) + x = BatchNorm(name=bn_name_base + '2a')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.Conv2D(nb_filter2, (kernel_size, kernel_size), padding='same', + name=conv_name_base + '2b', use_bias=use_bias)(x) + x = BatchNorm(name=bn_name_base + '2b')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.Conv2D(nb_filter3, (1, 1), name=conv_name_base + '2c', + use_bias=use_bias)(x) + x = BatchNorm(name=bn_name_base + '2c')(x, training=train_bn) + + x = KL.Add()([x, input_tensor]) + x = KL.Activation('relu', name='res' + str(stage) + block + '_out')(x) + return x + + +def conv_block(input_tensor, kernel_size, filters, stage, block, + strides=(2, 2), use_bias=True, train_bn=True): + """conv_block is the block that has a conv layer at shortcut + # Arguments + input_tensor: input tensor + kernel_size: default 3, the kernel size of middle conv layer at main path + filters: list of integers, the nb_filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + use_bias: Boolean. To use or not use a bias in conv layers. + train_bn: Boolean. Train or freeze Batch Norm layers + Note that from stage 3, the first conv layer at main path is with subsample=(2,2) + And the shortcut should have subsample=(2,2) as well + """ + nb_filter1, nb_filter2, nb_filter3 = filters + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = KL.Conv2D(nb_filter1, (1, 1), strides=strides, + name=conv_name_base + '2a', use_bias=use_bias)(input_tensor) + x = BatchNorm(name=bn_name_base + '2a')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.Conv2D(nb_filter2, (kernel_size, kernel_size), padding='same', + name=conv_name_base + '2b', use_bias=use_bias)(x) + x = BatchNorm(name=bn_name_base + '2b')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.Conv2D(nb_filter3, (1, 1), name=conv_name_base + + '2c', use_bias=use_bias)(x) + x = BatchNorm(name=bn_name_base + '2c')(x, training=train_bn) + + shortcut = KL.Conv2D(nb_filter3, (1, 1), strides=strides, + name=conv_name_base + '1', use_bias=use_bias)(input_tensor) + shortcut = BatchNorm(name=bn_name_base + '1')(shortcut, training=train_bn) + + x = KL.Add()([x, shortcut]) + x = KL.Activation('relu', name='res' + str(stage) + block + '_out')(x) + return x + + +def resnet_graph(input_image, architecture, stage5=False, train_bn=True): + """Build a ResNet graph. + architecture: Can be resnet50 or resnet101 + stage5: Boolean. If False, stage5 of the network is not created + train_bn: Boolean. Train or freeze Batch Norm layers + """ + assert architecture in ["resnet50", "resnet101"] + # Stage 1 + x = KL.ZeroPadding2D((3, 3))(input_image) + x = KL.Conv2D(64, (7, 7), strides=(2, 2), name='conv1', use_bias=True)(x) + x = BatchNorm(name='bn_conv1')(x, training=train_bn) + x = KL.Activation('relu')(x) + C1 = x = KL.MaxPooling2D((3, 3), strides=(2, 2), padding="same")(x) + # Stage 2 + x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1), train_bn=train_bn) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b', train_bn=train_bn) + C2 = x = identity_block(x, 3, [64, 64, 256], stage=2, block='c', train_bn=train_bn) + # Stage 3 + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a', train_bn=train_bn) + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b', train_bn=train_bn) + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c', train_bn=train_bn) + C3 = x = identity_block(x, 3, [128, 128, 512], stage=3, block='d', train_bn=train_bn) + # Stage 4 + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a', train_bn=train_bn) + block_count = {"resnet50": 5, "resnet101": 22}[architecture] + for i in range(block_count): + x = identity_block(x, 3, [256, 256, 1024], stage=4, block=chr(98 + i), train_bn=train_bn) + C4 = x + # Stage 5 + if stage5: + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a', train_bn=train_bn) + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b', train_bn=train_bn) + C5 = x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c', train_bn=train_bn) + else: + C5 = None + return [C1, C2, C3, C4, C5] + + +############################################################ +# Proposal Layer +############################################################ + +def apply_box_deltas_graph(boxes, deltas): + """Applies the given deltas to the given boxes. + boxes: [N, (y1, x1, y2, x2)] boxes to update + deltas: [N, (dy, dx, log(dh), log(dw))] refinements to apply + """ + # Convert to y, x, h, w + height = boxes[:, 2] - boxes[:, 0] + width = boxes[:, 3] - boxes[:, 1] + center_y = boxes[:, 0] + 0.5 * height + center_x = boxes[:, 1] + 0.5 * width + # Apply deltas + center_y += deltas[:, 0] * height + center_x += deltas[:, 1] * width + height *= tf.exp(deltas[:, 2]) + width *= tf.exp(deltas[:, 3]) + # Convert back to y1, x1, y2, x2 + y1 = center_y - 0.5 * height + x1 = center_x - 0.5 * width + y2 = y1 + height + x2 = x1 + width + result = tf.stack([y1, x1, y2, x2], axis=1, name="apply_box_deltas_out") + return result + + +def clip_boxes_graph(boxes, window): + """ + boxes: [N, (y1, x1, y2, x2)] + window: [4] in the form y1, x1, y2, x2 + """ + # Split + wy1, wx1, wy2, wx2 = tf.split(window, 4) + y1, x1, y2, x2 = tf.split(boxes, 4, axis=1) + # Clip + y1 = tf.maximum(tf.minimum(y1, wy2), wy1) + x1 = tf.maximum(tf.minimum(x1, wx2), wx1) + y2 = tf.maximum(tf.minimum(y2, wy2), wy1) + x2 = tf.maximum(tf.minimum(x2, wx2), wx1) + clipped = tf.concat([y1, x1, y2, x2], axis=1, name="clipped_boxes") + clipped.set_shape((clipped.shape[0], 4)) + return clipped + + +class ProposalLayer(KE.Layer): + """Receives anchor scores and selects a subset to pass as proposals + to the second stage. Filtering is done based on anchor scores and + non-max suppression to remove overlaps. It also applies bounding + box refinement deltas to anchors. + + Inputs: + rpn_probs: [batch, num_anchors, (bg prob, fg prob)] + rpn_bbox: [batch, num_anchors, (dy, dx, log(dh), log(dw))] + anchors: [batch, num_anchors, (y1, x1, y2, x2)] anchors in normalized coordinates + + Returns: + Proposals in normalized coordinates [batch, rois, (y1, x1, y2, x2)] + """ + + def __init__(self, proposal_count, nms_threshold, config=None, **kwargs): + super(ProposalLayer, self).__init__(**kwargs) + self.config = config + self.proposal_count = proposal_count + self.nms_threshold = nms_threshold + + def call(self, inputs): + # Box Scores. Use the foreground class confidence. [Batch, num_rois, 1] + scores = inputs[0][:, :, 1] + # Box deltas [batch, num_rois, 4] + deltas = inputs[1] + deltas = deltas * np.reshape(self.config.RPN_BBOX_STD_DEV, [1, 1, 4]) + # Anchors + anchors = inputs[2] + + # Improve performance by trimming to top anchors by score + # and doing the rest on the smaller subset. + pre_nms_limit = tf.minimum(self.config.PRE_NMS_LIMIT, tf.shape(anchors)[1]) + ix = tf.nn.top_k(scores, pre_nms_limit, sorted=True, + name="top_anchors").indices + scores = utils.batch_slice([scores, ix], lambda x, y: tf.gather(x, y), + self.config.IMAGES_PER_GPU) + deltas = utils.batch_slice([deltas, ix], lambda x, y: tf.gather(x, y), + self.config.IMAGES_PER_GPU) + pre_nms_anchors = utils.batch_slice([anchors, ix], lambda a, x: tf.gather(a, x), + self.config.IMAGES_PER_GPU, + names=["pre_nms_anchors"]) + + # Apply deltas to anchors to get refined anchors. + # [batch, N, (y1, x1, y2, x2)] + boxes = utils.batch_slice([pre_nms_anchors, deltas], + lambda x, y: apply_box_deltas_graph(x, y), + self.config.IMAGES_PER_GPU, + names=["refined_anchors"]) + + # Clip to image boundaries. Since we're in normalized coordinates, + # clip to 0..1 range. [batch, N, (y1, x1, y2, x2)] + window = np.array([0, 0, 1, 1], dtype=np.float32) + boxes = utils.batch_slice(boxes, + lambda x: clip_boxes_graph(x, window), + self.config.IMAGES_PER_GPU, + names=["refined_anchors_clipped"]) + + # Filter out small boxes + # According to Xinlei Chen's paper, this reduces detection accuracy + # for small objects, so we're skipping it. + + # Non-max suppression + def nms(boxes, scores): + indices = tf.image.non_max_suppression( + boxes, scores, self.proposal_count, + self.nms_threshold, name="rpn_non_max_suppression") + proposals = tf.gather(boxes, indices) + # Pad if needed + padding = tf.maximum(self.proposal_count - tf.shape(proposals)[0], 0) + proposals = tf.pad(proposals, [(0, padding), (0, 0)]) + return proposals + proposals = utils.batch_slice([boxes, scores], nms, + self.config.IMAGES_PER_GPU) + return proposals + + def compute_output_shape(self, input_shape): + return (None, self.proposal_count, 4) + + +############################################################ +# ROIAlign Layer +############################################################ + +def log2_graph(x): + """Implementation of Log2. TF doesn't have a native implementation.""" + return tf.log(x) / tf.log(2.0) + + +class PyramidROIAlign(KE.Layer): + """Implements ROI Pooling on multiple levels of the feature pyramid. + + Params: + - pool_shape: [pool_height, pool_width] of the output pooled regions. Usually [7, 7] + + Inputs: + - boxes: [batch, num_boxes, (y1, x1, y2, x2)] in normalized + coordinates. Possibly padded with zeros if not enough + boxes to fill the array. + - image_meta: [batch, (meta data)] Image details. See compose_image_meta() + - feature_maps: List of feature maps from different levels of the pyramid. + Each is [batch, height, width, channels] + + Output: + Pooled regions in the shape: [batch, num_boxes, pool_height, pool_width, channels]. + The width and height are those specific in the pool_shape in the layer + constructor. + """ + + def __init__(self, pool_shape, **kwargs): + super(PyramidROIAlign, self).__init__(**kwargs) + self.pool_shape = tuple(pool_shape) + + def call(self, inputs): + # Crop boxes [batch, num_boxes, (y1, x1, y2, x2)] in normalized coords + boxes = inputs[0] + + # Image meta + # Holds details about the image. See compose_image_meta() + image_meta = inputs[1] + + # Feature Maps. List of feature maps from different level of the + # feature pyramid. Each is [batch, height, width, channels] + feature_maps = inputs[2:] + + # Assign each ROI to a level in the pyramid based on the ROI area. + y1, x1, y2, x2 = tf.split(boxes, 4, axis=2) + h = y2 - y1 + w = x2 - x1 + # Use shape of first image. Images in a batch must have the same size. + image_shape = parse_image_meta_graph(image_meta)['image_shape'][0] + # Equation 1 in the Feature Pyramid Networks paper. Account for + # the fact that our coordinates are normalized here. + # e.g. a 224x224 ROI (in pixels) maps to P4 + image_area = tf.cast(image_shape[0] * image_shape[1], tf.float32) + roi_level = log2_graph(tf.sqrt(h * w) / (224.0 / tf.sqrt(image_area))) + roi_level = tf.minimum(5, tf.maximum( + 2, 4 + tf.cast(tf.round(roi_level), tf.int32))) + roi_level = tf.squeeze(roi_level, 2) + + # Loop through levels and apply ROI pooling to each. P2 to P5. + pooled = [] + box_to_level = [] + for i, level in enumerate(range(2, 6)): + ix = tf.where(tf.equal(roi_level, level)) + level_boxes = tf.gather_nd(boxes, ix) + + # Box indices for crop_and_resize. + box_indices = tf.cast(ix[:, 0], tf.int32) + + # Keep track of which box is mapped to which level + box_to_level.append(ix) + + # Stop gradient propogation to ROI proposals + level_boxes = tf.stop_gradient(level_boxes) + box_indices = tf.stop_gradient(box_indices) + + # Crop and Resize + # From Mask R-CNN paper: "We sample four regular locations, so + # that we can evaluate either max or average pooling. In fact, + # interpolating only a single value at each bin center (without + # pooling) is nearly as effective." + # + # Here we use the simplified approach of a single value per bin, + # which is how it's done in tf.crop_and_resize() + # Result: [batch * num_boxes, pool_height, pool_width, channels] + pooled.append(tf.image.crop_and_resize( + feature_maps[i], level_boxes, box_indices, self.pool_shape, + method="bilinear")) + + # Pack pooled features into one tensor + pooled = tf.concat(pooled, axis=0) + + # Pack box_to_level mapping into one array and add another + # column representing the order of pooled boxes + box_to_level = tf.concat(box_to_level, axis=0) + box_range = tf.expand_dims(tf.range(tf.shape(box_to_level)[0]), 1) + box_to_level = tf.concat([tf.cast(box_to_level, tf.int32), box_range], + axis=1) + + # Rearrange pooled features to match the order of the original boxes + # Sort box_to_level by batch then box index + # TF doesn't have a way to sort by two columns, so merge them and sort. + sorting_tensor = box_to_level[:, 0] * 100000 + box_to_level[:, 1] + ix = tf.nn.top_k(sorting_tensor, k=tf.shape( + box_to_level)[0]).indices[::-1] + ix = tf.gather(box_to_level[:, 2], ix) + pooled = tf.gather(pooled, ix) + + # Re-add the batch dimension + shape = tf.concat([tf.shape(boxes)[:2], tf.shape(pooled)[1:]], axis=0) + pooled = tf.reshape(pooled, shape) + return pooled + + def compute_output_shape(self, input_shape): + return input_shape[0][:2] + self.pool_shape + (input_shape[2][-1], ) + + +############################################################ +# Detection Target Layer +############################################################ + +def overlaps_graph(boxes1, boxes2): + """Computes IoU overlaps between two sets of boxes. + boxes1, boxes2: [N, (y1, x1, y2, x2)]. + """ + # 1. Tile boxes2 and repeat boxes1. This allows us to compare + # every boxes1 against every boxes2 without loops. + # TF doesn't have an equivalent to np.repeat() so simulate it + # using tf.tile() and tf.reshape. + b1 = tf.reshape(tf.tile(tf.expand_dims(boxes1, 1), + [1, 1, tf.shape(boxes2)[0]]), [-1, 4]) + b2 = tf.tile(boxes2, [tf.shape(boxes1)[0], 1]) + # 2. Compute intersections + b1_y1, b1_x1, b1_y2, b1_x2 = tf.split(b1, 4, axis=1) + b2_y1, b2_x1, b2_y2, b2_x2 = tf.split(b2, 4, axis=1) + y1 = tf.maximum(b1_y1, b2_y1) + x1 = tf.maximum(b1_x1, b2_x1) + y2 = tf.minimum(b1_y2, b2_y2) + x2 = tf.minimum(b1_x2, b2_x2) + intersection = tf.maximum(x2 - x1, 0) * tf.maximum(y2 - y1, 0) + # 3. Compute unions + b1_area = (b1_y2 - b1_y1) * (b1_x2 - b1_x1) + b2_area = (b2_y2 - b2_y1) * (b2_x2 - b2_x1) + union = b1_area + b2_area - intersection + # 4. Compute IoU and reshape to [boxes1, boxes2] + iou = intersection / union + overlaps = tf.reshape(iou, [tf.shape(boxes1)[0], tf.shape(boxes2)[0]]) + return overlaps + + +def detection_targets_graph(proposals, gt_class_ids, gt_boxes, gt_masks, config): + """Generates detection targets for one image. Subsamples proposals and + generates target class IDs, bounding box deltas, and masks for each. + + Inputs: + proposals: [POST_NMS_ROIS_TRAINING, (y1, x1, y2, x2)] in normalized coordinates. Might + be zero padded if there are not enough proposals. + gt_class_ids: [MAX_GT_INSTANCES] int class IDs + gt_boxes: [MAX_GT_INSTANCES, (y1, x1, y2, x2)] in normalized coordinates. + gt_masks: [height, width, MAX_GT_INSTANCES] of boolean type. + + Returns: Target ROIs and corresponding class IDs, bounding box shifts, + and masks. + rois: [TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)] in normalized coordinates + class_ids: [TRAIN_ROIS_PER_IMAGE]. Integer class IDs. Zero padded. + deltas: [TRAIN_ROIS_PER_IMAGE, (dy, dx, log(dh), log(dw))] + masks: [TRAIN_ROIS_PER_IMAGE, height, width]. Masks cropped to bbox + boundaries and resized to neural network output size. + + Note: Returned arrays might be zero padded if not enough target ROIs. + """ + # Assertions + asserts = [ + tf.Assert(tf.greater(tf.shape(proposals)[0], 0), [proposals], + name="roi_assertion"), + ] + with tf.control_dependencies(asserts): + proposals = tf.identity(proposals) + + # Remove zero padding + proposals, _ = trim_zeros_graph(proposals, name="trim_proposals") + gt_boxes, non_zeros = trim_zeros_graph(gt_boxes, name="trim_gt_boxes") + gt_class_ids = tf.boolean_mask(gt_class_ids, non_zeros, + name="trim_gt_class_ids") + gt_masks = tf.gather(gt_masks, tf.where(non_zeros)[:, 0], axis=2, + name="trim_gt_masks") + + # Handle COCO crowds + # A crowd box in COCO is a bounding box around several instances. Exclude + # them from training. A crowd box is given a negative class ID. + crowd_ix = tf.where(gt_class_ids < 0)[:, 0] + non_crowd_ix = tf.where(gt_class_ids > 0)[:, 0] + crowd_boxes = tf.gather(gt_boxes, crowd_ix) + gt_class_ids = tf.gather(gt_class_ids, non_crowd_ix) + gt_boxes = tf.gather(gt_boxes, non_crowd_ix) + gt_masks = tf.gather(gt_masks, non_crowd_ix, axis=2) + + # Compute overlaps matrix [proposals, gt_boxes] + overlaps = overlaps_graph(proposals, gt_boxes) + + # Compute overlaps with crowd boxes [proposals, crowd_boxes] + crowd_overlaps = overlaps_graph(proposals, crowd_boxes) + crowd_iou_max = tf.reduce_max(crowd_overlaps, axis=1) + no_crowd_bool = (crowd_iou_max < 0.001) + + # Determine positive and negative ROIs + roi_iou_max = tf.reduce_max(overlaps, axis=1) + # 1. Positive ROIs are those with >= 0.5 IoU with a GT box + positive_roi_bool = (roi_iou_max >= 0.5) + positive_indices = tf.where(positive_roi_bool)[:, 0] + # 2. Negative ROIs are those with < 0.5 with every GT box. Skip crowds. + negative_indices = tf.where(tf.logical_and(roi_iou_max < 0.5, no_crowd_bool))[:, 0] + + # Subsample ROIs. Aim for 33% positive + # Positive ROIs + positive_count = int(config.TRAIN_ROIS_PER_IMAGE * + config.ROI_POSITIVE_RATIO) + positive_indices = tf.random_shuffle(positive_indices)[:positive_count] + positive_count = tf.shape(positive_indices)[0] + # Negative ROIs. Add enough to maintain positive:negative ratio. + r = 1.0 / config.ROI_POSITIVE_RATIO + negative_count = tf.cast(r * tf.cast(positive_count, tf.float32), tf.int32) - positive_count + negative_indices = tf.random_shuffle(negative_indices)[:negative_count] + # Gather selected ROIs + positive_rois = tf.gather(proposals, positive_indices) + negative_rois = tf.gather(proposals, negative_indices) + + # Assign positive ROIs to GT boxes. + positive_overlaps = tf.gather(overlaps, positive_indices) + roi_gt_box_assignment = tf.cond( + tf.greater(tf.shape(positive_overlaps)[1], 0), + true_fn = lambda: tf.argmax(positive_overlaps, axis=1), + false_fn = lambda: tf.cast(tf.constant([]),tf.int64) + ) + roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment) + roi_gt_class_ids = tf.gather(gt_class_ids, roi_gt_box_assignment) + + # Compute bbox refinement for positive ROIs + deltas = utils.box_refinement_graph(positive_rois, roi_gt_boxes) + deltas /= config.BBOX_STD_DEV + + # Assign positive ROIs to GT masks + # Permute masks to [N, height, width, 1] + transposed_masks = tf.expand_dims(tf.transpose(gt_masks, [2, 0, 1]), -1) + # Pick the right mask for each ROI + roi_masks = tf.gather(transposed_masks, roi_gt_box_assignment) + + # Compute mask targets + boxes = positive_rois + if config.USE_MINI_MASK: + # Transform ROI coordinates from normalized image space + # to normalized mini-mask space. + y1, x1, y2, x2 = tf.split(positive_rois, 4, axis=1) + gt_y1, gt_x1, gt_y2, gt_x2 = tf.split(roi_gt_boxes, 4, axis=1) + gt_h = gt_y2 - gt_y1 + gt_w = gt_x2 - gt_x1 + y1 = (y1 - gt_y1) / gt_h + x1 = (x1 - gt_x1) / gt_w + y2 = (y2 - gt_y1) / gt_h + x2 = (x2 - gt_x1) / gt_w + boxes = tf.concat([y1, x1, y2, x2], 1) + box_ids = tf.range(0, tf.shape(roi_masks)[0]) + masks = tf.image.crop_and_resize(tf.cast(roi_masks, tf.float32), boxes, + box_ids, + config.MASK_SHAPE) + # Remove the extra dimension from masks. + masks = tf.squeeze(masks, axis=3) + + # Threshold mask pixels at 0.5 to have GT masks be 0 or 1 to use with + # binary cross entropy loss. + masks = tf.round(masks) + + # Append negative ROIs and pad bbox deltas and masks that + # are not used for negative ROIs with zeros. + rois = tf.concat([positive_rois, negative_rois], axis=0) + N = tf.shape(negative_rois)[0] + P = tf.maximum(config.TRAIN_ROIS_PER_IMAGE - tf.shape(rois)[0], 0) + rois = tf.pad(rois, [(0, P), (0, 0)]) + roi_gt_boxes = tf.pad(roi_gt_boxes, [(0, N + P), (0, 0)]) + roi_gt_class_ids = tf.pad(roi_gt_class_ids, [(0, N + P)]) + deltas = tf.pad(deltas, [(0, N + P), (0, 0)]) + masks = tf.pad(masks, [[0, N + P], (0, 0), (0, 0)]) + + return rois, roi_gt_class_ids, deltas, masks + + +class DetectionTargetLayer(KE.Layer): + """Subsamples proposals and generates target box refinement, class_ids, + and masks for each. + + Inputs: + proposals: [batch, N, (y1, x1, y2, x2)] in normalized coordinates. Might + be zero padded if there are not enough proposals. + gt_class_ids: [batch, MAX_GT_INSTANCES] Integer class IDs. + gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] in normalized + coordinates. + gt_masks: [batch, height, width, MAX_GT_INSTANCES] of boolean type + + Returns: Target ROIs and corresponding class IDs, bounding box shifts, + and masks. + rois: [batch, TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)] in normalized + coordinates + target_class_ids: [batch, TRAIN_ROIS_PER_IMAGE]. Integer class IDs. + target_deltas: [batch, TRAIN_ROIS_PER_IMAGE, (dy, dx, log(dh), log(dw)] + target_mask: [batch, TRAIN_ROIS_PER_IMAGE, height, width] + Masks cropped to bbox boundaries and resized to neural + network output size. + + Note: Returned arrays might be zero padded if not enough target ROIs. + """ + + def __init__(self, config, **kwargs): + super(DetectionTargetLayer, self).__init__(**kwargs) + self.config = config + + def call(self, inputs): + proposals = inputs[0] + gt_class_ids = inputs[1] + gt_boxes = inputs[2] + gt_masks = inputs[3] + + # Slice the batch and run a graph for each slice + # TODO: Rename target_bbox to target_deltas for clarity + names = ["rois", "target_class_ids", "target_bbox", "target_mask"] + outputs = utils.batch_slice( + [proposals, gt_class_ids, gt_boxes, gt_masks], + lambda w, x, y, z: detection_targets_graph( + w, x, y, z, self.config), + self.config.IMAGES_PER_GPU, names=names) + return outputs + + def compute_output_shape(self, input_shape): + return [ + (None, self.config.TRAIN_ROIS_PER_IMAGE, 4), # rois + (None, self.config.TRAIN_ROIS_PER_IMAGE), # class_ids + (None, self.config.TRAIN_ROIS_PER_IMAGE, 4), # deltas + (None, self.config.TRAIN_ROIS_PER_IMAGE, self.config.MASK_SHAPE[0], + self.config.MASK_SHAPE[1]) # masks + ] + + def compute_mask(self, inputs, mask=None): + return [None, None, None, None] + + +############################################################ +# Detection Layer +############################################################ + +def refine_detections_graph(rois, probs, deltas, window, config): + """Refine classified proposals and filter overlaps and return final + detections. + + Inputs: + rois: [N, (y1, x1, y2, x2)] in normalized coordinates + probs: [N, num_classes]. Class probabilities. + deltas: [N, num_classes, (dy, dx, log(dh), log(dw))]. Class-specific + bounding box deltas. + window: (y1, x1, y2, x2) in normalized coordinates. The part of the image + that contains the image excluding the padding. + + Returns detections shaped: [num_detections, (y1, x1, y2, x2, class_id, score)] where + coordinates are normalized. + """ + # Class IDs per ROI + class_ids = tf.argmax(probs, axis=1, output_type=tf.int32) + # Class probability of the top class of each ROI + indices = tf.stack([tf.range(probs.shape[0]), class_ids], axis=1) + class_scores = tf.gather_nd(probs, indices) + # Class-specific bounding box deltas + deltas_specific = tf.gather_nd(deltas, indices) + # Apply bounding box deltas + # Shape: [boxes, (y1, x1, y2, x2)] in normalized coordinates + refined_rois = apply_box_deltas_graph( + rois, deltas_specific * config.BBOX_STD_DEV) + # Clip boxes to image window + refined_rois = clip_boxes_graph(refined_rois, window) + + # TODO: Filter out boxes with zero area + + # Filter out background boxes + keep = tf.where(class_ids > 0)[:, 0] + # Filter out low confidence boxes + if config.DETECTION_MIN_CONFIDENCE: + conf_keep = tf.where(class_scores >= config.DETECTION_MIN_CONFIDENCE)[:, 0] + keep = tf.sets.set_intersection(tf.expand_dims(keep, 0), + tf.expand_dims(conf_keep, 0)) + keep = tf.sparse_tensor_to_dense(keep)[0] + + # Apply per-class NMS + # 1. Prepare variables + pre_nms_class_ids = tf.gather(class_ids, keep) + pre_nms_scores = tf.gather(class_scores, keep) + pre_nms_rois = tf.gather(refined_rois, keep) + unique_pre_nms_class_ids = tf.unique(pre_nms_class_ids)[0] + + def nms_keep_map(class_id): + """Apply Non-Maximum Suppression on ROIs of the given class.""" + # Indices of ROIs of the given class + ixs = tf.where(tf.equal(pre_nms_class_ids, class_id))[:, 0] + # Apply NMS + class_keep = tf.image.non_max_suppression( + tf.gather(pre_nms_rois, ixs), + tf.gather(pre_nms_scores, ixs), + max_output_size=config.DETECTION_MAX_INSTANCES, + iou_threshold=config.DETECTION_NMS_THRESHOLD) + # Map indices + class_keep = tf.gather(keep, tf.gather(ixs, class_keep)) + # Pad with -1 so returned tensors have the same shape + gap = config.DETECTION_MAX_INSTANCES - tf.shape(class_keep)[0] + class_keep = tf.pad(class_keep, [(0, gap)], + mode='CONSTANT', constant_values=-1) + # Set shape so map_fn() can infer result shape + class_keep.set_shape([config.DETECTION_MAX_INSTANCES]) + return class_keep + + # 2. Map over class IDs + nms_keep = tf.map_fn(nms_keep_map, unique_pre_nms_class_ids, + dtype=tf.int64) + # 3. Merge results into one list, and remove -1 padding + nms_keep = tf.reshape(nms_keep, [-1]) + nms_keep = tf.gather(nms_keep, tf.where(nms_keep > -1)[:, 0]) + # 4. Compute intersection between keep and nms_keep + keep = tf.sets.set_intersection(tf.expand_dims(keep, 0), + tf.expand_dims(nms_keep, 0)) + keep = tf.sparse_tensor_to_dense(keep)[0] + # Keep top detections + roi_count = config.DETECTION_MAX_INSTANCES + class_scores_keep = tf.gather(class_scores, keep) + num_keep = tf.minimum(tf.shape(class_scores_keep)[0], roi_count) + top_ids = tf.nn.top_k(class_scores_keep, k=num_keep, sorted=True)[1] + keep = tf.gather(keep, top_ids) + + # Arrange output as [N, (y1, x1, y2, x2, class_id, score)] + # Coordinates are normalized. + detections = tf.concat([ + tf.gather(refined_rois, keep), + tf.to_float(tf.gather(class_ids, keep))[..., tf.newaxis], + tf.gather(class_scores, keep)[..., tf.newaxis] + ], axis=1) + + # Pad with zeros if detections < DETECTION_MAX_INSTANCES + gap = config.DETECTION_MAX_INSTANCES - tf.shape(detections)[0] + detections = tf.pad(detections, [(0, gap), (0, 0)], "CONSTANT") + return detections + + +class DetectionLayer(KE.Layer): + """Takes classified proposal boxes and their bounding box deltas and + returns the final detection boxes. + + Returns: + [batch, num_detections, (y1, x1, y2, x2, class_id, class_score)] where + coordinates are normalized. + """ + + def __init__(self, config=None, **kwargs): + super(DetectionLayer, self).__init__(**kwargs) + self.config = config + + def call(self, inputs): + rois = inputs[0] + mrcnn_class = inputs[1] + mrcnn_bbox = inputs[2] + image_meta = inputs[3] + + # Get windows of images in normalized coordinates. Windows are the area + # in the image that excludes the padding. + # Use the shape of the first image in the batch to normalize the window + # because we know that all images get resized to the same size. + m = parse_image_meta_graph(image_meta) + image_shape = m['image_shape'][0] + window = norm_boxes_graph(m['window'], image_shape[:2]) + + # Run detection refinement graph on each item in the batch + detections_batch = utils.batch_slice( + [rois, mrcnn_class, mrcnn_bbox, window], + lambda x, y, w, z: refine_detections_graph(x, y, w, z, self.config), + self.config.IMAGES_PER_GPU) + + # Reshape output + # [batch, num_detections, (y1, x1, y2, x2, class_id, class_score)] in + # normalized coordinates + return tf.reshape( + detections_batch, + [self.config.BATCH_SIZE, self.config.DETECTION_MAX_INSTANCES, 6]) + + def compute_output_shape(self, input_shape): + return (None, self.config.DETECTION_MAX_INSTANCES, 6) + + +############################################################ +# Region Proposal Network (RPN) +############################################################ + +def rpn_graph(feature_map, anchors_per_location, anchor_stride): + """Builds the computation graph of Region Proposal Network. + + feature_map: backbone features [batch, height, width, depth] + anchors_per_location: number of anchors per pixel in the feature map + anchor_stride: Controls the density of anchors. Typically 1 (anchors for + every pixel in the feature map), or 2 (every other pixel). + + Returns: + rpn_class_logits: [batch, H * W * anchors_per_location, 2] Anchor classifier logits (before softmax) + rpn_probs: [batch, H * W * anchors_per_location, 2] Anchor classifier probabilities. + rpn_bbox: [batch, H * W * anchors_per_location, (dy, dx, log(dh), log(dw))] Deltas to be + applied to anchors. + """ + # TODO: check if stride of 2 causes alignment issues if the feature map + # is not even. + # Shared convolutional base of the RPN + shared = KL.Conv2D(512, (3, 3), padding='same', activation='relu', + strides=anchor_stride, + name='rpn_conv_shared')(feature_map) + + # Anchor Score. [batch, height, width, anchors per location * 2]. + x = KL.Conv2D(2 * anchors_per_location, (1, 1), padding='valid', + activation='linear', name='rpn_class_raw')(shared) + + # Reshape to [batch, anchors, 2] + rpn_class_logits = KL.Lambda( + lambda t: tf.reshape(t, [tf.shape(t)[0], -1, 2]))(x) + + # Softmax on last dimension of BG/FG. + rpn_probs = KL.Activation( + "softmax", name="rpn_class_xxx")(rpn_class_logits) + + # Bounding box refinement. [batch, H, W, anchors per location * depth] + # where depth is [x, y, log(w), log(h)] + x = KL.Conv2D(anchors_per_location * 4, (1, 1), padding="valid", + activation='linear', name='rpn_bbox_pred')(shared) + + # Reshape to [batch, anchors, 4] + rpn_bbox = KL.Lambda(lambda t: tf.reshape(t, [tf.shape(t)[0], -1, 4]))(x) + + return [rpn_class_logits, rpn_probs, rpn_bbox] + + +def build_rpn_model(anchor_stride, anchors_per_location, depth): + """Builds a Keras model of the Region Proposal Network. + It wraps the RPN graph so it can be used multiple times with shared + weights. + + anchors_per_location: number of anchors per pixel in the feature map + anchor_stride: Controls the density of anchors. Typically 1 (anchors for + every pixel in the feature map), or 2 (every other pixel). + depth: Depth of the backbone feature map. + + Returns a Keras Model object. The model outputs, when called, are: + rpn_class_logits: [batch, H * W * anchors_per_location, 2] Anchor classifier logits (before softmax) + rpn_probs: [batch, H * W * anchors_per_location, 2] Anchor classifier probabilities. + rpn_bbox: [batch, H * W * anchors_per_location, (dy, dx, log(dh), log(dw))] Deltas to be + applied to anchors. + """ + input_feature_map = KL.Input(shape=[None, None, depth], + name="input_rpn_feature_map") + outputs = rpn_graph(input_feature_map, anchors_per_location, anchor_stride) + return KM.Model([input_feature_map], outputs, name="rpn_model") + + +############################################################ +# Feature Pyramid Network Heads +############################################################ + +def fpn_classifier_graph(rois, feature_maps, image_meta, + pool_size, num_classes, train_bn=True, + fc_layers_size=1024): + """Builds the computation graph of the feature pyramid network classifier + and regressor heads. + + rois: [batch, num_rois, (y1, x1, y2, x2)] Proposal boxes in normalized + coordinates. + feature_maps: List of feature maps from different layers of the pyramid, + [P2, P3, P4, P5]. Each has a different resolution. + image_meta: [batch, (meta data)] Image details. See compose_image_meta() + pool_size: The width of the square feature map generated from ROI Pooling. + num_classes: number of classes, which determines the depth of the results + train_bn: Boolean. Train or freeze Batch Norm layers + fc_layers_size: Size of the 2 FC layers + + Returns: + logits: [batch, num_rois, NUM_CLASSES] classifier logits (before softmax) + probs: [batch, num_rois, NUM_CLASSES] classifier probabilities + bbox_deltas: [batch, num_rois, NUM_CLASSES, (dy, dx, log(dh), log(dw))] Deltas to apply to + proposal boxes + """ + # ROI Pooling + # Shape: [batch, num_rois, POOL_SIZE, POOL_SIZE, channels] + x = PyramidROIAlign([pool_size, pool_size], + name="roi_align_classifier")([rois, image_meta] + feature_maps) + # Two 1024 FC layers (implemented with Conv2D for consistency) + x = KL.TimeDistributed(KL.Conv2D(fc_layers_size, (pool_size, pool_size), padding="valid"), + name="mrcnn_class_conv1")(x) + x = KL.TimeDistributed(BatchNorm(), name='mrcnn_class_bn1')(x, training=train_bn) + x = KL.Activation('relu')(x) + x = KL.TimeDistributed(KL.Conv2D(fc_layers_size, (1, 1)), + name="mrcnn_class_conv2")(x) + x = KL.TimeDistributed(BatchNorm(), name='mrcnn_class_bn2')(x, training=train_bn) + x = KL.Activation('relu')(x) + + shared = KL.Lambda(lambda x: K.squeeze(K.squeeze(x, 3), 2), + name="pool_squeeze")(x) + + # Classifier head + mrcnn_class_logits = KL.TimeDistributed(KL.Dense(num_classes), + name='mrcnn_class_logits')(shared) + mrcnn_probs = KL.TimeDistributed(KL.Activation("softmax"), + name="mrcnn_class")(mrcnn_class_logits) + + # BBox head + # [batch, num_rois, NUM_CLASSES * (dy, dx, log(dh), log(dw))] + x = KL.TimeDistributed(KL.Dense(num_classes * 4, activation='linear'), + name='mrcnn_bbox_fc')(shared) + # Reshape to [batch, num_rois, NUM_CLASSES, (dy, dx, log(dh), log(dw))] + s = K.int_shape(x) + mrcnn_bbox = KL.Reshape((s[1], num_classes, 4), name="mrcnn_bbox")(x) + + return mrcnn_class_logits, mrcnn_probs, mrcnn_bbox + + +def build_fpn_mask_graph(rois, feature_maps, image_meta, + pool_size, num_classes, train_bn=True): + """Builds the computation graph of the mask head of Feature Pyramid Network. + + rois: [batch, num_rois, (y1, x1, y2, x2)] Proposal boxes in normalized + coordinates. + feature_maps: List of feature maps from different layers of the pyramid, + [P2, P3, P4, P5]. Each has a different resolution. + image_meta: [batch, (meta data)] Image details. See compose_image_meta() + pool_size: The width of the square feature map generated from ROI Pooling. + num_classes: number of classes, which determines the depth of the results + train_bn: Boolean. Train or freeze Batch Norm layers + + Returns: Masks [batch, num_rois, MASK_POOL_SIZE, MASK_POOL_SIZE, NUM_CLASSES] + """ + # ROI Pooling + # Shape: [batch, num_rois, MASK_POOL_SIZE, MASK_POOL_SIZE, channels] + x = PyramidROIAlign([pool_size, pool_size], + name="roi_align_mask")([rois, image_meta] + feature_maps) + + # Conv layers + x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"), + name="mrcnn_mask_conv1")(x) + x = KL.TimeDistributed(BatchNorm(), + name='mrcnn_mask_bn1')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"), + name="mrcnn_mask_conv2")(x) + x = KL.TimeDistributed(BatchNorm(), + name='mrcnn_mask_bn2')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"), + name="mrcnn_mask_conv3")(x) + x = KL.TimeDistributed(BatchNorm(), + name='mrcnn_mask_bn3')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"), + name="mrcnn_mask_conv4")(x) + x = KL.TimeDistributed(BatchNorm(), + name='mrcnn_mask_bn4')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.TimeDistributed(KL.Conv2DTranspose(256, (2, 2), strides=2, activation="relu"), + name="mrcnn_mask_deconv")(x) + x = KL.TimeDistributed(KL.Conv2D(num_classes, (1, 1), strides=1, activation="sigmoid"), + name="mrcnn_mask")(x) + return x + + +############################################################ +# Loss Functions +############################################################ + +def smooth_l1_loss(y_true, y_pred): + """Implements Smooth-L1 loss. + y_true and y_pred are typically: [N, 4], but could be any shape. + """ + diff = K.abs(y_true - y_pred) + less_than_one = K.cast(K.less(diff, 1.0), "float32") + loss = (less_than_one * 0.5 * diff**2) + (1 - less_than_one) * (diff - 0.5) + return loss + + +def rpn_class_loss_graph(rpn_match, rpn_class_logits): + """RPN anchor classifier loss. + + rpn_match: [batch, anchors, 1]. Anchor match type. 1=positive, + -1=negative, 0=neutral anchor. + rpn_class_logits: [batch, anchors, 2]. RPN classifier logits for BG/FG. + """ + # Squeeze last dim to simplify + rpn_match = tf.squeeze(rpn_match, -1) + # Get anchor classes. Convert the -1/+1 match to 0/1 values. + anchor_class = K.cast(K.equal(rpn_match, 1), tf.int32) + # Positive and Negative anchors contribute to the loss, + # but neutral anchors (match value = 0) don't. + indices = tf.where(K.not_equal(rpn_match, 0)) + # Pick rows that contribute to the loss and filter out the rest. + rpn_class_logits = tf.gather_nd(rpn_class_logits, indices) + anchor_class = tf.gather_nd(anchor_class, indices) + # Cross entropy loss + loss = K.sparse_categorical_crossentropy(target=anchor_class, + output=rpn_class_logits, + from_logits=True) + loss = K.switch(tf.size(loss) > 0, K.mean(loss), tf.constant(0.0)) + return loss + + +def rpn_bbox_loss_graph(config, target_bbox, rpn_match, rpn_bbox): + """Return the RPN bounding box loss graph. + + config: the model config object. + target_bbox: [batch, max positive anchors, (dy, dx, log(dh), log(dw))]. + Uses 0 padding to fill in unsed bbox deltas. + rpn_match: [batch, anchors, 1]. Anchor match type. 1=positive, + -1=negative, 0=neutral anchor. + rpn_bbox: [batch, anchors, (dy, dx, log(dh), log(dw))] + """ + # Positive anchors contribute to the loss, but negative and + # neutral anchors (match value of 0 or -1) don't. + rpn_match = K.squeeze(rpn_match, -1) + indices = tf.where(K.equal(rpn_match, 1)) + + # Pick bbox deltas that contribute to the loss + rpn_bbox = tf.gather_nd(rpn_bbox, indices) + + # Trim target bounding box deltas to the same length as rpn_bbox. + batch_counts = K.sum(K.cast(K.equal(rpn_match, 1), tf.int32), axis=1) + target_bbox = batch_pack_graph(target_bbox, batch_counts, + config.IMAGES_PER_GPU) + + loss = smooth_l1_loss(target_bbox, rpn_bbox) + + loss = K.switch(tf.size(loss) > 0, K.mean(loss), tf.constant(0.0)) + return loss + + +def mrcnn_class_loss_graph(target_class_ids, pred_class_logits, + active_class_ids): + """Loss for the classifier head of Mask RCNN. + + target_class_ids: [batch, num_rois]. Integer class IDs. Uses zero + padding to fill in the array. + pred_class_logits: [batch, num_rois, num_classes] + active_class_ids: [batch, num_classes]. Has a value of 1 for + classes that are in the dataset of the image, and 0 + for classes that are not in the dataset. + """ + # During model building, Keras calls this function with + # target_class_ids of type float32. Unclear why. Cast it + # to int to get around it. + target_class_ids = tf.cast(target_class_ids, 'int64') + + # Find predictions of classes that are not in the dataset. + pred_class_ids = tf.argmax(pred_class_logits, axis=2) + # TODO: Update this line to work with batch > 1. Right now it assumes all + # images in a batch have the same active_class_ids + pred_active = tf.gather(active_class_ids[0], pred_class_ids) + + # Loss + loss = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=target_class_ids, logits=pred_class_logits) + + # Erase losses of predictions of classes that are not in the active + # classes of the image. + loss = loss * pred_active + + # Computer loss mean. Use only predictions that contribute + # to the loss to get a correct mean. + loss = tf.reduce_sum(loss) / tf.reduce_sum(pred_active) + return loss + + +def mrcnn_bbox_loss_graph(target_bbox, target_class_ids, pred_bbox): + """Loss for Mask R-CNN bounding box refinement. + + target_bbox: [batch, num_rois, (dy, dx, log(dh), log(dw))] + target_class_ids: [batch, num_rois]. Integer class IDs. + pred_bbox: [batch, num_rois, num_classes, (dy, dx, log(dh), log(dw))] + """ + # Reshape to merge batch and roi dimensions for simplicity. + target_class_ids = K.reshape(target_class_ids, (-1,)) + target_bbox = K.reshape(target_bbox, (-1, 4)) + pred_bbox = K.reshape(pred_bbox, (-1, K.int_shape(pred_bbox)[2], 4)) + + # Only positive ROIs contribute to the loss. And only + # the right class_id of each ROI. Get their indices. + positive_roi_ix = tf.where(target_class_ids > 0)[:, 0] + positive_roi_class_ids = tf.cast( + tf.gather(target_class_ids, positive_roi_ix), tf.int64) + indices = tf.stack([positive_roi_ix, positive_roi_class_ids], axis=1) + + # Gather the deltas (predicted and true) that contribute to loss + target_bbox = tf.gather(target_bbox, positive_roi_ix) + pred_bbox = tf.gather_nd(pred_bbox, indices) + + # Smooth-L1 Loss + loss = K.switch(tf.size(target_bbox) > 0, + smooth_l1_loss(y_true=target_bbox, y_pred=pred_bbox), + tf.constant(0.0)) + loss = K.mean(loss) + return loss + + +def mrcnn_mask_loss_graph(target_masks, target_class_ids, pred_masks): + """Mask binary cross-entropy loss for the masks head. + + target_masks: [batch, num_rois, height, width]. + A float32 tensor of values 0 or 1. Uses zero padding to fill array. + target_class_ids: [batch, num_rois]. Integer class IDs. Zero padded. + pred_masks: [batch, proposals, height, width, num_classes] float32 tensor + with values from 0 to 1. + """ + # Reshape for simplicity. Merge first two dimensions into one. + target_class_ids = K.reshape(target_class_ids, (-1,)) + mask_shape = tf.shape(target_masks) + target_masks = K.reshape(target_masks, (-1, mask_shape[2], mask_shape[3])) + pred_shape = tf.shape(pred_masks) + pred_masks = K.reshape(pred_masks, + (-1, pred_shape[2], pred_shape[3], pred_shape[4])) + # Permute predicted masks to [N, num_classes, height, width] + pred_masks = tf.transpose(pred_masks, [0, 3, 1, 2]) + + # Only positive ROIs contribute to the loss. And only + # the class specific mask of each ROI. + positive_ix = tf.where(target_class_ids > 0)[:, 0] + positive_class_ids = tf.cast( + tf.gather(target_class_ids, positive_ix), tf.int64) + indices = tf.stack([positive_ix, positive_class_ids], axis=1) + + # Gather the masks (predicted and true) that contribute to loss + y_true = tf.gather(target_masks, positive_ix) + y_pred = tf.gather_nd(pred_masks, indices) + + # Compute binary cross entropy. If no positive ROIs, then return 0. + # shape: [batch, roi, num_classes] + loss = K.switch(tf.size(y_true) > 0, + K.binary_crossentropy(target=y_true, output=y_pred), + tf.constant(0.0)) + loss = K.mean(loss) + return loss + + +############################################################ +# Data Generator +############################################################ + +def load_image_gt(dataset, config, image_id, augment=False, augmentation=None, + use_mini_mask=False): + """Load and return ground truth data for an image (image, mask, bounding boxes). + + augment: (deprecated. Use augmentation instead). If true, apply random + image augmentation. Currently, only horizontal flipping is offered. + augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) augmentation. + For example, passing imgaug.augmenters.Fliplr(0.5) flips images + right/left 50% of the time. + use_mini_mask: If False, returns full-size masks that are the same height + and width as the original image. These can be big, for example + 1024x1024x100 (for 100 instances). Mini masks are smaller, typically, + 224x224 and are generated by extracting the bounding box of the + object and resizing it to MINI_MASK_SHAPE. + + Returns: + image: [height, width, 3] + shape: the original shape of the image before resizing and cropping. + class_ids: [instance_count] Integer class IDs + bbox: [instance_count, (y1, x1, y2, x2)] + mask: [height, width, instance_count]. The height and width are those + of the image unless use_mini_mask is True, in which case they are + defined in MINI_MASK_SHAPE. + """ + # Load image and mask + image = dataset.load_image(image_id) + mask, class_ids = dataset.load_mask(image_id) + original_shape = image.shape + image, window, scale, padding, crop = utils.resize_image( + image, + min_dim=config.IMAGE_MIN_DIM, + min_scale=config.IMAGE_MIN_SCALE, + max_dim=config.IMAGE_MAX_DIM, + mode=config.IMAGE_RESIZE_MODE) + mask = utils.resize_mask(mask, scale, padding, crop) + + # Random horizontal flips. + # TODO: will be removed in a future update in favor of augmentation + if augment: + logging.warning("'augment' is deprecated. Use 'augmentation' instead.") + if random.randint(0, 1): + image = np.fliplr(image) + mask = np.fliplr(mask) + + # Augmentation + # This requires the imgaug lib (https://github.com/aleju/imgaug) + if augmentation: + import imgaug + + # Augmenters that are safe to apply to masks + # Some, such as Affine, have settings that make them unsafe, so always + # test your augmentation on masks + MASK_AUGMENTERS = ["Sequential", "SomeOf", "OneOf", "Sometimes", + "Fliplr", "Flipud", "CropAndPad", + "Affine", "PiecewiseAffine"] + + def hook(images, augmenter, parents, default): + """Determines which augmenters to apply to masks.""" + return augmenter.__class__.__name__ in MASK_AUGMENTERS + + # Store shapes before augmentation to compare + image_shape = image.shape + mask_shape = mask.shape + # Make augmenters deterministic to apply similarly to images and masks + det = augmentation.to_deterministic() + image = det.augment_image(image) + # Change mask to np.uint8 because imgaug doesn't support np.bool + mask = det.augment_image(mask.astype(np.uint8), + hooks=imgaug.HooksImages(activator=hook)) + # Verify that shapes didn't change + assert image.shape == image_shape, "Augmentation shouldn't change image size" + assert mask.shape == mask_shape, "Augmentation shouldn't change mask size" + # Change mask back to bool + mask = mask.astype(np.bool) + + # Note that some boxes might be all zeros if the corresponding mask got cropped out. + # and here is to filter them out + _idx = np.sum(mask, axis=(0, 1)) > 0 + mask = mask[:, :, _idx] + class_ids = class_ids[_idx] + # Bounding boxes. Note that some boxes might be all zeros + # if the corresponding mask got cropped out. + # bbox: [num_instances, (y1, x1, y2, x2)] + bbox = utils.extract_bboxes(mask) + + # Active classes + # Different datasets have different classes, so track the + # classes supported in the dataset of this image. + active_class_ids = np.zeros([dataset.num_classes], dtype=np.int32) + source_class_ids = dataset.source_class_ids[dataset.image_info[image_id]["source"]] + active_class_ids[source_class_ids] = 1 + + # Resize masks to smaller size to reduce memory usage + if use_mini_mask: + mask = utils.minimize_mask(bbox, mask, config.MINI_MASK_SHAPE) + + # Image meta data + image_meta = compose_image_meta(image_id, original_shape, image.shape, + window, scale, active_class_ids) + + return image, image_meta, class_ids, bbox, mask + + +def build_detection_targets(rpn_rois, gt_class_ids, gt_boxes, gt_masks, config): + """Generate targets for training Stage 2 classifier and mask heads. + This is not used in normal training. It's useful for debugging or to train + the Mask RCNN heads without using the RPN head. + + Inputs: + rpn_rois: [N, (y1, x1, y2, x2)] proposal boxes. + gt_class_ids: [instance count] Integer class IDs + gt_boxes: [instance count, (y1, x1, y2, x2)] + gt_masks: [height, width, instance count] Ground truth masks. Can be full + size or mini-masks. + + Returns: + rois: [TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)] + class_ids: [TRAIN_ROIS_PER_IMAGE]. Integer class IDs. + bboxes: [TRAIN_ROIS_PER_IMAGE, NUM_CLASSES, (y, x, log(h), log(w))]. Class-specific + bbox refinements. + masks: [TRAIN_ROIS_PER_IMAGE, height, width, NUM_CLASSES). Class specific masks cropped + to bbox boundaries and resized to neural network output size. + """ + assert rpn_rois.shape[0] > 0 + assert gt_class_ids.dtype == np.int32, "Expected int but got {}".format( + gt_class_ids.dtype) + assert gt_boxes.dtype == np.int32, "Expected int but got {}".format( + gt_boxes.dtype) + assert gt_masks.dtype == np.bool_, "Expected bool but got {}".format( + gt_masks.dtype) + + # It's common to add GT Boxes to ROIs but we don't do that here because + # according to XinLei Chen's paper, it doesn't help. + + # Trim empty padding in gt_boxes and gt_masks parts + instance_ids = np.where(gt_class_ids > 0)[0] + assert instance_ids.shape[0] > 0, "Image must contain instances." + gt_class_ids = gt_class_ids[instance_ids] + gt_boxes = gt_boxes[instance_ids] + gt_masks = gt_masks[:, :, instance_ids] + + # Compute areas of ROIs and ground truth boxes. + rpn_roi_area = (rpn_rois[:, 2] - rpn_rois[:, 0]) * \ + (rpn_rois[:, 3] - rpn_rois[:, 1]) + gt_box_area = (gt_boxes[:, 2] - gt_boxes[:, 0]) * \ + (gt_boxes[:, 3] - gt_boxes[:, 1]) + + # Compute overlaps [rpn_rois, gt_boxes] + overlaps = np.zeros((rpn_rois.shape[0], gt_boxes.shape[0])) + for i in range(overlaps.shape[1]): + gt = gt_boxes[i] + overlaps[:, i] = utils.compute_iou( + gt, rpn_rois, gt_box_area[i], rpn_roi_area) + + # Assign ROIs to GT boxes + rpn_roi_iou_argmax = np.argmax(overlaps, axis=1) + rpn_roi_iou_max = overlaps[np.arange( + overlaps.shape[0]), rpn_roi_iou_argmax] + # GT box assigned to each ROI + rpn_roi_gt_boxes = gt_boxes[rpn_roi_iou_argmax] + rpn_roi_gt_class_ids = gt_class_ids[rpn_roi_iou_argmax] + + # Positive ROIs are those with >= 0.5 IoU with a GT box. + fg_ids = np.where(rpn_roi_iou_max > 0.5)[0] + + # Negative ROIs are those with max IoU 0.1-0.5 (hard example mining) + # TODO: To hard example mine or not to hard example mine, that's the question + # bg_ids = np.where((rpn_roi_iou_max >= 0.1) & (rpn_roi_iou_max < 0.5))[0] + bg_ids = np.where(rpn_roi_iou_max < 0.5)[0] + + # Subsample ROIs. Aim for 33% foreground. + # FG + fg_roi_count = int(config.TRAIN_ROIS_PER_IMAGE * config.ROI_POSITIVE_RATIO) + if fg_ids.shape[0] > fg_roi_count: + keep_fg_ids = np.random.choice(fg_ids, fg_roi_count, replace=False) + else: + keep_fg_ids = fg_ids + # BG + remaining = config.TRAIN_ROIS_PER_IMAGE - keep_fg_ids.shape[0] + if bg_ids.shape[0] > remaining: + keep_bg_ids = np.random.choice(bg_ids, remaining, replace=False) + else: + keep_bg_ids = bg_ids + # Combine indices of ROIs to keep + keep = np.concatenate([keep_fg_ids, keep_bg_ids]) + # Need more? + remaining = config.TRAIN_ROIS_PER_IMAGE - keep.shape[0] + if remaining > 0: + # Looks like we don't have enough samples to maintain the desired + # balance. Reduce requirements and fill in the rest. This is + # likely different from the Mask RCNN paper. + + # There is a small chance we have neither fg nor bg samples. + if keep.shape[0] == 0: + # Pick bg regions with easier IoU threshold + bg_ids = np.where(rpn_roi_iou_max < 0.5)[0] + assert bg_ids.shape[0] >= remaining + keep_bg_ids = np.random.choice(bg_ids, remaining, replace=False) + assert keep_bg_ids.shape[0] == remaining + keep = np.concatenate([keep, keep_bg_ids]) + else: + # Fill the rest with repeated bg rois. + keep_extra_ids = np.random.choice( + keep_bg_ids, remaining, replace=True) + keep = np.concatenate([keep, keep_extra_ids]) + assert keep.shape[0] == config.TRAIN_ROIS_PER_IMAGE, \ + "keep doesn't match ROI batch size {}, {}".format( + keep.shape[0], config.TRAIN_ROIS_PER_IMAGE) + + # Reset the gt boxes assigned to BG ROIs. + rpn_roi_gt_boxes[keep_bg_ids, :] = 0 + rpn_roi_gt_class_ids[keep_bg_ids] = 0 + + # For each kept ROI, assign a class_id, and for FG ROIs also add bbox refinement. + rois = rpn_rois[keep] + roi_gt_boxes = rpn_roi_gt_boxes[keep] + roi_gt_class_ids = rpn_roi_gt_class_ids[keep] + roi_gt_assignment = rpn_roi_iou_argmax[keep] + + # Class-aware bbox deltas. [y, x, log(h), log(w)] + bboxes = np.zeros((config.TRAIN_ROIS_PER_IMAGE, + config.NUM_CLASSES, 4), dtype=np.float32) + pos_ids = np.where(roi_gt_class_ids > 0)[0] + bboxes[pos_ids, roi_gt_class_ids[pos_ids]] = utils.box_refinement( + rois[pos_ids], roi_gt_boxes[pos_ids, :4]) + # Normalize bbox refinements + bboxes /= config.BBOX_STD_DEV + + # Generate class-specific target masks + masks = np.zeros((config.TRAIN_ROIS_PER_IMAGE, config.MASK_SHAPE[0], config.MASK_SHAPE[1], config.NUM_CLASSES), + dtype=np.float32) + for i in pos_ids: + class_id = roi_gt_class_ids[i] + assert class_id > 0, "class id must be greater than 0" + gt_id = roi_gt_assignment[i] + class_mask = gt_masks[:, :, gt_id] + + if config.USE_MINI_MASK: + # Create a mask placeholder, the size of the image + placeholder = np.zeros(config.IMAGE_SHAPE[:2], dtype=bool) + # GT box + gt_y1, gt_x1, gt_y2, gt_x2 = gt_boxes[gt_id] + gt_w = gt_x2 - gt_x1 + gt_h = gt_y2 - gt_y1 + # Resize mini mask to size of GT box + placeholder[gt_y1:gt_y2, gt_x1:gt_x2] = \ + np.round(utils.resize(class_mask, (gt_h, gt_w))).astype(bool) + # Place the mini batch in the placeholder + class_mask = placeholder + + # Pick part of the mask and resize it + y1, x1, y2, x2 = rois[i].astype(np.int32) + m = class_mask[y1:y2, x1:x2] + mask = utils.resize(m, config.MASK_SHAPE) + masks[i, :, :, class_id] = mask + + return rois, roi_gt_class_ids, bboxes, masks + + +def build_rpn_targets(image_shape, anchors, gt_class_ids, gt_boxes, config): + """Given the anchors and GT boxes, compute overlaps and identify positive + anchors and deltas to refine them to match their corresponding GT boxes. + + anchors: [num_anchors, (y1, x1, y2, x2)] + gt_class_ids: [num_gt_boxes] Integer class IDs. + gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] + + Returns: + rpn_match: [N] (int32) matches between anchors and GT boxes. + 1 = positive anchor, -1 = negative anchor, 0 = neutral + rpn_bbox: [N, (dy, dx, log(dh), log(dw))] Anchor bbox deltas. + """ + # RPN Match: 1 = positive anchor, -1 = negative anchor, 0 = neutral + rpn_match = np.zeros([anchors.shape[0]], dtype=np.int32) + # RPN bounding boxes: [max anchors per image, (dy, dx, log(dh), log(dw))] + rpn_bbox = np.zeros((config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4)) + + # Handle COCO crowds + # A crowd box in COCO is a bounding box around several instances. Exclude + # them from training. A crowd box is given a negative class ID. + crowd_ix = np.where(gt_class_ids < 0)[0] + if crowd_ix.shape[0] > 0: + # Filter out crowds from ground truth class IDs and boxes + non_crowd_ix = np.where(gt_class_ids > 0)[0] + crowd_boxes = gt_boxes[crowd_ix] + gt_class_ids = gt_class_ids[non_crowd_ix] + gt_boxes = gt_boxes[non_crowd_ix] + # Compute overlaps with crowd boxes [anchors, crowds] + crowd_overlaps = utils.compute_overlaps(anchors, crowd_boxes) + crowd_iou_max = np.amax(crowd_overlaps, axis=1) + no_crowd_bool = (crowd_iou_max < 0.001) + else: + # All anchors don't intersect a crowd + no_crowd_bool = np.ones([anchors.shape[0]], dtype=bool) + + # Compute overlaps [num_anchors, num_gt_boxes] + overlaps = utils.compute_overlaps(anchors, gt_boxes) + + # Match anchors to GT Boxes + # If an anchor overlaps a GT box with IoU >= 0.7 then it's positive. + # If an anchor overlaps a GT box with IoU < 0.3 then it's negative. + # Neutral anchors are those that don't match the conditions above, + # and they don't influence the loss function. + # However, don't keep any GT box unmatched (rare, but happens). Instead, + # match it to the closest anchor (even if its max IoU is < 0.3). + # + # 1. Set negative anchors first. They get overwritten below if a GT box is + # matched to them. Skip boxes in crowd areas. + anchor_iou_argmax = np.argmax(overlaps, axis=1) + anchor_iou_max = overlaps[np.arange(overlaps.shape[0]), anchor_iou_argmax] + rpn_match[(anchor_iou_max < 0.3) & (no_crowd_bool)] = -1 + # 2. Set an anchor for each GT box (regardless of IoU value). + # If multiple anchors have the same IoU match all of them + gt_iou_argmax = np.argwhere(overlaps == np.max(overlaps, axis=0))[:,0] + rpn_match[gt_iou_argmax] = 1 + # 3. Set anchors with high overlap as positive. + rpn_match[anchor_iou_max >= 0.7] = 1 + + # Subsample to balance positive and negative anchors + # Don't let positives be more than half the anchors + ids = np.where(rpn_match == 1)[0] + extra = len(ids) - (config.RPN_TRAIN_ANCHORS_PER_IMAGE // 2) + if extra > 0: + # Reset the extra ones to neutral + ids = np.random.choice(ids, extra, replace=False) + rpn_match[ids] = 0 + # Same for negative proposals + ids = np.where(rpn_match == -1)[0] + extra = len(ids) - (config.RPN_TRAIN_ANCHORS_PER_IMAGE - + np.sum(rpn_match == 1)) + if extra > 0: + # Rest the extra ones to neutral + ids = np.random.choice(ids, extra, replace=False) + rpn_match[ids] = 0 + + # For positive anchors, compute shift and scale needed to transform them + # to match the corresponding GT boxes. + ids = np.where(rpn_match == 1)[0] + ix = 0 # index into rpn_bbox + # TODO: use box_refinement() rather than duplicating the code here + for i, a in zip(ids, anchors[ids]): + # Closest gt box (it might have IoU < 0.7) + gt = gt_boxes[anchor_iou_argmax[i]] + + # Convert coordinates to center plus width/height. + # GT Box + gt_h = gt[2] - gt[0] + gt_w = gt[3] - gt[1] + gt_center_y = gt[0] + 0.5 * gt_h + gt_center_x = gt[1] + 0.5 * gt_w + # Anchor + a_h = a[2] - a[0] + a_w = a[3] - a[1] + a_center_y = a[0] + 0.5 * a_h + a_center_x = a[1] + 0.5 * a_w + + # Compute the bbox refinement that the RPN should predict. + rpn_bbox[ix] = [ + (gt_center_y - a_center_y) / a_h, + (gt_center_x - a_center_x) / a_w, + np.log(gt_h / a_h), + np.log(gt_w / a_w), + ] + # Normalize + rpn_bbox[ix] /= config.RPN_BBOX_STD_DEV + ix += 1 + + return rpn_match, rpn_bbox + + +def generate_random_rois(image_shape, count, gt_class_ids, gt_boxes): + """Generates ROI proposals similar to what a region proposal network + would generate. + + image_shape: [Height, Width, Depth] + count: Number of ROIs to generate + gt_class_ids: [N] Integer ground truth class IDs + gt_boxes: [N, (y1, x1, y2, x2)] Ground truth boxes in pixels. + + Returns: [count, (y1, x1, y2, x2)] ROI boxes in pixels. + """ + # placeholder + rois = np.zeros((count, 4), dtype=np.int32) + + # Generate random ROIs around GT boxes (90% of count) + rois_per_box = int(0.9 * count / gt_boxes.shape[0]) + for i in range(gt_boxes.shape[0]): + gt_y1, gt_x1, gt_y2, gt_x2 = gt_boxes[i] + h = gt_y2 - gt_y1 + w = gt_x2 - gt_x1 + # random boundaries + r_y1 = max(gt_y1 - h, 0) + r_y2 = min(gt_y2 + h, image_shape[0]) + r_x1 = max(gt_x1 - w, 0) + r_x2 = min(gt_x2 + w, image_shape[1]) + + # To avoid generating boxes with zero area, we generate double what + # we need and filter out the extra. If we get fewer valid boxes + # than we need, we loop and try again. + while True: + y1y2 = np.random.randint(r_y1, r_y2, (rois_per_box * 2, 2)) + x1x2 = np.random.randint(r_x1, r_x2, (rois_per_box * 2, 2)) + # Filter out zero area boxes + threshold = 1 + y1y2 = y1y2[np.abs(y1y2[:, 0] - y1y2[:, 1]) >= + threshold][:rois_per_box] + x1x2 = x1x2[np.abs(x1x2[:, 0] - x1x2[:, 1]) >= + threshold][:rois_per_box] + if y1y2.shape[0] == rois_per_box and x1x2.shape[0] == rois_per_box: + break + + # Sort on axis 1 to ensure x1 <= x2 and y1 <= y2 and then reshape + # into x1, y1, x2, y2 order + x1, x2 = np.split(np.sort(x1x2, axis=1), 2, axis=1) + y1, y2 = np.split(np.sort(y1y2, axis=1), 2, axis=1) + box_rois = np.hstack([y1, x1, y2, x2]) + rois[rois_per_box * i:rois_per_box * (i + 1)] = box_rois + + # Generate random ROIs anywhere in the image (10% of count) + remaining_count = count - (rois_per_box * gt_boxes.shape[0]) + # To avoid generating boxes with zero area, we generate double what + # we need and filter out the extra. If we get fewer valid boxes + # than we need, we loop and try again. + while True: + y1y2 = np.random.randint(0, image_shape[0], (remaining_count * 2, 2)) + x1x2 = np.random.randint(0, image_shape[1], (remaining_count * 2, 2)) + # Filter out zero area boxes + threshold = 1 + y1y2 = y1y2[np.abs(y1y2[:, 0] - y1y2[:, 1]) >= + threshold][:remaining_count] + x1x2 = x1x2[np.abs(x1x2[:, 0] - x1x2[:, 1]) >= + threshold][:remaining_count] + if y1y2.shape[0] == remaining_count and x1x2.shape[0] == remaining_count: + break + + # Sort on axis 1 to ensure x1 <= x2 and y1 <= y2 and then reshape + # into x1, y1, x2, y2 order + x1, x2 = np.split(np.sort(x1x2, axis=1), 2, axis=1) + y1, y2 = np.split(np.sort(y1y2, axis=1), 2, axis=1) + global_rois = np.hstack([y1, x1, y2, x2]) + rois[-remaining_count:] = global_rois + return rois + + +def data_generator(dataset, config, shuffle=True, augment=False, augmentation=None, + random_rois=0, batch_size=1, detection_targets=False, + no_augmentation_sources=None): + """A generator that returns images and corresponding target class ids, + bounding box deltas, and masks. + + dataset: The Dataset object to pick data from + config: The model config object + shuffle: If True, shuffles the samples before every epoch + augment: (deprecated. Use augmentation instead). If true, apply random + image augmentation. Currently, only horizontal flipping is offered. + augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) augmentation. + For example, passing imgaug.augmenters.Fliplr(0.5) flips images + right/left 50% of the time. + random_rois: If > 0 then generate proposals to be used to train the + network classifier and mask heads. Useful if training + the Mask RCNN part without the RPN. + batch_size: How many images to return in each call + detection_targets: If True, generate detection targets (class IDs, bbox + deltas, and masks). Typically for debugging or visualizations because + in trainig detection targets are generated by DetectionTargetLayer. + no_augmentation_sources: Optional. List of sources to exclude for + augmentation. A source is string that identifies a dataset and is + defined in the Dataset class. + + Returns a Python generator. Upon calling next() on it, the + generator returns two lists, inputs and outputs. The contents + of the lists differs depending on the received arguments: + inputs list: + - images: [batch, H, W, C] + - image_meta: [batch, (meta data)] Image details. See compose_image_meta() + - rpn_match: [batch, N] Integer (1=positive anchor, -1=negative, 0=neutral) + - rpn_bbox: [batch, N, (dy, dx, log(dh), log(dw))] Anchor bbox deltas. + - gt_class_ids: [batch, MAX_GT_INSTANCES] Integer class IDs + - gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] + - gt_masks: [batch, height, width, MAX_GT_INSTANCES]. The height and width + are those of the image unless use_mini_mask is True, in which + case they are defined in MINI_MASK_SHAPE. + + outputs list: Usually empty in regular training. But if detection_targets + is True then the outputs list contains target class_ids, bbox deltas, + and masks. + """ + b = 0 # batch item index + image_index = -1 + image_ids = np.copy(dataset.image_ids) + error_count = 0 + no_augmentation_sources = no_augmentation_sources or [] + + # Anchors + # [anchor_count, (y1, x1, y2, x2)] + backbone_shapes = compute_backbone_shapes(config, config.IMAGE_SHAPE) + anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES, + config.RPN_ANCHOR_RATIOS, + backbone_shapes, + config.BACKBONE_STRIDES, + config.RPN_ANCHOR_STRIDE) + + # Keras requires a generator to run indefinitely. + while True: + try: + # Increment index to pick next image. Shuffle if at the start of an epoch. + image_index = (image_index + 1) % len(image_ids) + if shuffle and image_index == 0: + np.random.shuffle(image_ids) + + # Get GT bounding boxes and masks for image. + image_id = image_ids[image_index] + + # If the image source is not to be augmented pass None as augmentation + if dataset.image_info[image_id]['source'] in no_augmentation_sources: + image, image_meta, gt_class_ids, gt_boxes, gt_masks = \ + load_image_gt(dataset, config, image_id, augment=augment, + augmentation=None, + use_mini_mask=config.USE_MINI_MASK) + else: + image, image_meta, gt_class_ids, gt_boxes, gt_masks = \ + load_image_gt(dataset, config, image_id, augment=augment, + augmentation=augmentation, + use_mini_mask=config.USE_MINI_MASK) + + # Skip images that have no instances. This can happen in cases + # where we train on a subset of classes and the image doesn't + # have any of the classes we care about. + if not np.any(gt_class_ids > 0): + continue + + # RPN Targets + rpn_match, rpn_bbox = build_rpn_targets(image.shape, anchors, + gt_class_ids, gt_boxes, config) + + # Mask R-CNN Targets + if random_rois: + rpn_rois = generate_random_rois( + image.shape, random_rois, gt_class_ids, gt_boxes) + if detection_targets: + rois, mrcnn_class_ids, mrcnn_bbox, mrcnn_mask =\ + build_detection_targets( + rpn_rois, gt_class_ids, gt_boxes, gt_masks, config) + + # Init batch arrays + if b == 0: + batch_image_meta = np.zeros( + (batch_size,) + image_meta.shape, dtype=image_meta.dtype) + batch_rpn_match = np.zeros( + [batch_size, anchors.shape[0], 1], dtype=rpn_match.dtype) + batch_rpn_bbox = np.zeros( + [batch_size, config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4], dtype=rpn_bbox.dtype) + batch_images = np.zeros( + (batch_size,) + image.shape, dtype=np.float32) + batch_gt_class_ids = np.zeros( + (batch_size, config.MAX_GT_INSTANCES), dtype=np.int32) + batch_gt_boxes = np.zeros( + (batch_size, config.MAX_GT_INSTANCES, 4), dtype=np.int32) + batch_gt_masks = np.zeros( + (batch_size, gt_masks.shape[0], gt_masks.shape[1], + config.MAX_GT_INSTANCES), dtype=gt_masks.dtype) + if random_rois: + batch_rpn_rois = np.zeros( + (batch_size, rpn_rois.shape[0], 4), dtype=rpn_rois.dtype) + if detection_targets: + batch_rois = np.zeros( + (batch_size,) + rois.shape, dtype=rois.dtype) + batch_mrcnn_class_ids = np.zeros( + (batch_size,) + mrcnn_class_ids.shape, dtype=mrcnn_class_ids.dtype) + batch_mrcnn_bbox = np.zeros( + (batch_size,) + mrcnn_bbox.shape, dtype=mrcnn_bbox.dtype) + batch_mrcnn_mask = np.zeros( + (batch_size,) + mrcnn_mask.shape, dtype=mrcnn_mask.dtype) + + # If more instances than fits in the array, sub-sample from them. + if gt_boxes.shape[0] > config.MAX_GT_INSTANCES: + ids = np.random.choice( + np.arange(gt_boxes.shape[0]), config.MAX_GT_INSTANCES, replace=False) + gt_class_ids = gt_class_ids[ids] + gt_boxes = gt_boxes[ids] + gt_masks = gt_masks[:, :, ids] + + # Add to batch + batch_image_meta[b] = image_meta + batch_rpn_match[b] = rpn_match[:, np.newaxis] + batch_rpn_bbox[b] = rpn_bbox + batch_images[b] = mold_image(image.astype(np.float32), config) + batch_gt_class_ids[b, :gt_class_ids.shape[0]] = gt_class_ids + batch_gt_boxes[b, :gt_boxes.shape[0]] = gt_boxes + batch_gt_masks[b, :, :, :gt_masks.shape[-1]] = gt_masks + if random_rois: + batch_rpn_rois[b] = rpn_rois + if detection_targets: + batch_rois[b] = rois + batch_mrcnn_class_ids[b] = mrcnn_class_ids + batch_mrcnn_bbox[b] = mrcnn_bbox + batch_mrcnn_mask[b] = mrcnn_mask + b += 1 + + # Batch full? + if b >= batch_size: + inputs = [batch_images, batch_image_meta, batch_rpn_match, batch_rpn_bbox, + batch_gt_class_ids, batch_gt_boxes, batch_gt_masks] + outputs = [] + + if random_rois: + inputs.extend([batch_rpn_rois]) + if detection_targets: + inputs.extend([batch_rois]) + # Keras requires that output and targets have the same number of dimensions + batch_mrcnn_class_ids = np.expand_dims( + batch_mrcnn_class_ids, -1) + outputs.extend( + [batch_mrcnn_class_ids, batch_mrcnn_bbox, batch_mrcnn_mask]) + + yield inputs, outputs + + # start a new batch + b = 0 + except (GeneratorExit, KeyboardInterrupt): + raise + except: + # Log it and skip the image + logging.exception("Error processing image {}".format( + dataset.image_info[image_id])) + error_count += 1 + if error_count > 5: + raise + + +############################################################ +# MaskRCNN Class +############################################################ + +class MaskRCNN(): + """Encapsulates the Mask RCNN model functionality. + + The actual Keras model is in the keras_model property. + """ + + def __init__(self, mode, config, model_dir): + """ + mode: Either "training" or "inference" + config: A Sub-class of the Config class + model_dir: Directory to save training logs and trained weights + """ + assert mode in ['training', 'inference'] + self.mode = mode + self.config = config + self.model_dir = model_dir + self.set_log_dir() + self.keras_model = self.build(mode=mode, config=config) + + def build(self, mode, config): + """Build Mask R-CNN architecture. + input_shape: The shape of the input image. + mode: Either "training" or "inference". The inputs and + outputs of the model differ accordingly. + """ + assert mode in ['training', 'inference'] + + # Image size must be dividable by 2 multiple times + h, w = config.IMAGE_SHAPE[:2] + if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6): + raise Exception("Image size must be dividable by 2 at least 6 times " + "to avoid fractions when downscaling and upscaling." + "For example, use 256, 320, 384, 448, 512, ... etc. ") + + # Inputs + input_image = KL.Input( + shape=[None, None, config.IMAGE_SHAPE[2]], name="input_image") + input_image_meta = KL.Input(shape=[config.IMAGE_META_SIZE], + name="input_image_meta") + if mode == "training": + # RPN GT + input_rpn_match = KL.Input( + shape=[None, 1], name="input_rpn_match", dtype=tf.int32) + input_rpn_bbox = KL.Input( + shape=[None, 4], name="input_rpn_bbox", dtype=tf.float32) + + # Detection GT (class IDs, bounding boxes, and masks) + # 1. GT Class IDs (zero padded) + input_gt_class_ids = KL.Input( + shape=[None], name="input_gt_class_ids", dtype=tf.int32) + # 2. GT Boxes in pixels (zero padded) + # [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] in image coordinates + input_gt_boxes = KL.Input( + shape=[None, 4], name="input_gt_boxes", dtype=tf.float32) + # Normalize coordinates + gt_boxes = KL.Lambda(lambda x: norm_boxes_graph( + x, K.shape(input_image)[1:3]))(input_gt_boxes) + # 3. GT Masks (zero padded) + # [batch, height, width, MAX_GT_INSTANCES] + if config.USE_MINI_MASK: + input_gt_masks = KL.Input( + shape=[config.MINI_MASK_SHAPE[0], + config.MINI_MASK_SHAPE[1], None], + name="input_gt_masks", dtype=bool) + else: + input_gt_masks = KL.Input( + shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None], + name="input_gt_masks", dtype=bool) + elif mode == "inference": + # Anchors in normalized coordinates + input_anchors = KL.Input(shape=[None, 4], name="input_anchors") + + # Build the shared convolutional layers. + # Bottom-up Layers + # Returns a list of the last layers of each stage, 5 in total. + # Don't create the thead (stage 5), so we pick the 4th item in the list. + if callable(config.BACKBONE): + _, C2, C3, C4, C5 = config.BACKBONE(input_image, stage5=True, + train_bn=config.TRAIN_BN) + else: + _, C2, C3, C4, C5 = resnet_graph(input_image, config.BACKBONE, + stage5=True, train_bn=config.TRAIN_BN) + # Top-down Layers + # TODO: add assert to varify feature map sizes match what's in config + P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5) + P4 = KL.Add(name="fpn_p4add")([ + KL.UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5), + KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4)]) + P3 = KL.Add(name="fpn_p3add")([ + KL.UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4), + KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3)]) + P2 = KL.Add(name="fpn_p2add")([ + KL.UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3), + KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2)]) + # Attach 3x3 conv to all P layers to get the final feature maps. + P2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p2")(P2) + P3 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p3")(P3) + P4 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p4")(P4) + P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p5")(P5) + # P6 is used for the 5th anchor scale in RPN. Generated by + # subsampling from P5 with stride of 2. + P6 = KL.MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5) + + # Note that P6 is used in RPN, but not in the classifier heads. + rpn_feature_maps = [P2, P3, P4, P5, P6] + mrcnn_feature_maps = [P2, P3, P4, P5] + + # Anchors + if mode == "training": + anchors = self.get_anchors(config.IMAGE_SHAPE) + # Duplicate across the batch dimension because Keras requires it + # TODO: can this be optimized to avoid duplicating the anchors? + anchors = np.broadcast_to(anchors, (config.BATCH_SIZE,) + anchors.shape) + # A hack to get around Keras's bad support for constants + anchors = KL.Lambda(lambda x: tf.Variable(anchors), name="anchors")(input_image) + else: + anchors = input_anchors + + # RPN Model + rpn = build_rpn_model(config.RPN_ANCHOR_STRIDE, + len(config.RPN_ANCHOR_RATIOS), config.TOP_DOWN_PYRAMID_SIZE) + # Loop through pyramid layers + layer_outputs = [] # list of lists + for p in rpn_feature_maps: + layer_outputs.append(rpn([p])) + # Concatenate layer outputs + # Convert from list of lists of level outputs to list of lists + # of outputs across levels. + # e.g. [[a1, b1, c1], [a2, b2, c2]] => [[a1, a2], [b1, b2], [c1, c2]] + output_names = ["rpn_class_logits", "rpn_class", "rpn_bbox"] + outputs = list(zip(*layer_outputs)) + outputs = [KL.Concatenate(axis=1, name=n)(list(o)) + for o, n in zip(outputs, output_names)] + + rpn_class_logits, rpn_class, rpn_bbox = outputs + + # Generate proposals + # Proposals are [batch, N, (y1, x1, y2, x2)] in normalized coordinates + # and zero padded. + proposal_count = config.POST_NMS_ROIS_TRAINING if mode == "training"\ + else config.POST_NMS_ROIS_INFERENCE + rpn_rois = ProposalLayer( + proposal_count=proposal_count, + nms_threshold=config.RPN_NMS_THRESHOLD, + name="ROI", + config=config)([rpn_class, rpn_bbox, anchors]) + + if mode == "training": + # Class ID mask to mark class IDs supported by the dataset the image + # came from. + active_class_ids = KL.Lambda( + lambda x: parse_image_meta_graph(x)["active_class_ids"] + )(input_image_meta) + + if not config.USE_RPN_ROIS: + # Ignore predicted ROIs and use ROIs provided as an input. + input_rois = KL.Input(shape=[config.POST_NMS_ROIS_TRAINING, 4], + name="input_roi", dtype=np.int32) + # Normalize coordinates + target_rois = KL.Lambda(lambda x: norm_boxes_graph( + x, K.shape(input_image)[1:3]))(input_rois) + else: + target_rois = rpn_rois + + # Generate detection targets + # Subsamples proposals and generates target outputs for training + # Note that proposal class IDs, gt_boxes, and gt_masks are zero + # padded. Equally, returned rois and targets are zero padded. + rois, target_class_ids, target_bbox, target_mask =\ + DetectionTargetLayer(config, name="proposal_targets")([ + target_rois, input_gt_class_ids, gt_boxes, input_gt_masks]) + + # Network Heads + # TODO: verify that this handles zero padded ROIs + mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\ + fpn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta, + config.POOL_SIZE, config.NUM_CLASSES, + train_bn=config.TRAIN_BN, + fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) + + mrcnn_mask = build_fpn_mask_graph(rois, mrcnn_feature_maps, + input_image_meta, + config.MASK_POOL_SIZE, + config.NUM_CLASSES, + train_bn=config.TRAIN_BN) + + # TODO: clean up (use tf.identify if necessary) + output_rois = KL.Lambda(lambda x: x * 1, name="output_rois")(rois) + + # Losses + rpn_class_loss = KL.Lambda(lambda x: rpn_class_loss_graph(*x), name="rpn_class_loss")( + [input_rpn_match, rpn_class_logits]) + rpn_bbox_loss = KL.Lambda(lambda x: rpn_bbox_loss_graph(config, *x), name="rpn_bbox_loss")( + [input_rpn_bbox, input_rpn_match, rpn_bbox]) + class_loss = KL.Lambda(lambda x: mrcnn_class_loss_graph(*x), name="mrcnn_class_loss")( + [target_class_ids, mrcnn_class_logits, active_class_ids]) + bbox_loss = KL.Lambda(lambda x: mrcnn_bbox_loss_graph(*x), name="mrcnn_bbox_loss")( + [target_bbox, target_class_ids, mrcnn_bbox]) + mask_loss = KL.Lambda(lambda x: mrcnn_mask_loss_graph(*x), name="mrcnn_mask_loss")( + [target_mask, target_class_ids, mrcnn_mask]) + + # Model + inputs = [input_image, input_image_meta, + input_rpn_match, input_rpn_bbox, input_gt_class_ids, input_gt_boxes, input_gt_masks] + if not config.USE_RPN_ROIS: + inputs.append(input_rois) + outputs = [rpn_class_logits, rpn_class, rpn_bbox, + mrcnn_class_logits, mrcnn_class, mrcnn_bbox, mrcnn_mask, + rpn_rois, output_rois, + rpn_class_loss, rpn_bbox_loss, class_loss, bbox_loss, mask_loss] + model = KM.Model(inputs, outputs, name='mask_rcnn') + else: + # Network Heads + # Proposal classifier and BBox regressor heads + mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\ + fpn_classifier_graph(rpn_rois, mrcnn_feature_maps, input_image_meta, + config.POOL_SIZE, config.NUM_CLASSES, + train_bn=config.TRAIN_BN, + fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) + + # Detections + # output is [batch, num_detections, (y1, x1, y2, x2, class_id, score)] in + # normalized coordinates + detections = DetectionLayer(config, name="mrcnn_detection")( + [rpn_rois, mrcnn_class, mrcnn_bbox, input_image_meta]) + + # Create masks for detections + detection_boxes = KL.Lambda(lambda x: x[..., :4])(detections) + mrcnn_mask = build_fpn_mask_graph(detection_boxes, mrcnn_feature_maps, + input_image_meta, + config.MASK_POOL_SIZE, + config.NUM_CLASSES, + train_bn=config.TRAIN_BN) + + model = KM.Model([input_image, input_image_meta, input_anchors], + [detections, mrcnn_class, mrcnn_bbox, + mrcnn_mask, rpn_rois, rpn_class, rpn_bbox], + name='mask_rcnn') + + # Add multi-GPU support. + if config.GPU_COUNT > 1: + from mrcnn.parallel_model import ParallelModel + model = ParallelModel(model, config.GPU_COUNT) + + return model + + def find_last(self): + """Finds the last checkpoint file of the last trained model in the + model directory. + Returns: + The path of the last checkpoint file + """ + # Get directory names. Each directory corresponds to a model + dir_names = next(os.walk(self.model_dir))[1] + key = self.config.NAME.lower() + dir_names = filter(lambda f: f.startswith(key), dir_names) + dir_names = sorted(dir_names) + if not dir_names: + import errno + raise FileNotFoundError( + errno.ENOENT, + "Could not find model directory under {}".format(self.model_dir)) + # Pick last directory + dir_name = os.path.join(self.model_dir, dir_names[-1]) + # Find the last checkpoint + checkpoints = next(os.walk(dir_name))[2] + checkpoints = filter(lambda f: f.startswith("mask_rcnn"), checkpoints) + checkpoints = sorted(checkpoints) + if not checkpoints: + import errno + raise FileNotFoundError( + errno.ENOENT, "Could not find weight files in {}".format(dir_name)) + checkpoint = os.path.join(dir_name, checkpoints[-1]) + return checkpoint + + def load_weights(self, filepath, by_name=False, exclude=None): + """Modified version of the corresponding Keras function with + the addition of multi-GPU support and the ability to exclude + some layers from loading. + exclude: list of layer names to exclude + """ + import h5py + # Conditional import to support versions of Keras before 2.2 + # TODO: remove in about 6 months (end of 2018) + try: + from keras.engine import saving + except ImportError: + # Keras before 2.2 used the 'topology' namespace. + from keras.engine import topology as saving + + if exclude: + by_name = True + + if h5py is None: + raise ImportError('`load_weights` requires h5py.') + f = h5py.File(filepath, mode='r') + if 'layer_names' not in f.attrs and 'model_weights' in f: + f = f['model_weights'] + + # In multi-GPU training, we wrap the model. Get layers + # of the inner model because they have the weights. + keras_model = self.keras_model + layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\ + else keras_model.layers + + # Exclude some layers + if exclude: + layers = filter(lambda l: l.name not in exclude, layers) + + if by_name: + saving.load_weights_from_hdf5_group_by_name(f, layers) + else: + saving.load_weights_from_hdf5_group(f, layers) + if hasattr(f, 'close'): + f.close() + + # Update the log directory + self.set_log_dir(filepath) + + def get_imagenet_weights(self): + """Downloads ImageNet trained weights from Keras. + Returns path to weights file. + """ + from keras.utils.data_utils import get_file + TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/'\ + 'releases/download/v0.2/'\ + 'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' + weights_path = get_file('resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5', + TF_WEIGHTS_PATH_NO_TOP, + cache_subdir='models', + md5_hash='a268eb855778b3df3c7506639542a6af') + return weights_path + + def compile(self, learning_rate, momentum): + """Gets the model ready for training. Adds losses, regularization, and + metrics. Then calls the Keras compile() function. + """ + # Optimizer object + optimizer = keras.optimizers.SGD( + lr=learning_rate, momentum=momentum, + clipnorm=self.config.GRADIENT_CLIP_NORM) + # Add Losses + # First, clear previously set losses to avoid duplication + self.keras_model._losses = [] + self.keras_model._per_input_losses = {} + loss_names = [ + "rpn_class_loss", "rpn_bbox_loss", + "mrcnn_class_loss", "mrcnn_bbox_loss", "mrcnn_mask_loss"] + for name in loss_names: + layer = self.keras_model.get_layer(name) + if layer.output in self.keras_model.losses: + continue + loss = ( + tf.reduce_mean(layer.output, keepdims=True) + * self.config.LOSS_WEIGHTS.get(name, 1.)) + self.keras_model.add_loss(loss) + + # Add L2 Regularization + # Skip gamma and beta weights of batch normalization layers. + reg_losses = [ + keras.regularizers.l2(self.config.WEIGHT_DECAY)(w) / tf.cast(tf.size(w), tf.float32) + for w in self.keras_model.trainable_weights + if 'gamma' not in w.name and 'beta' not in w.name] + self.keras_model.add_loss(tf.add_n(reg_losses)) + + # Compile + self.keras_model.compile( + optimizer=optimizer, + loss=[None] * len(self.keras_model.outputs)) + + # Add metrics for losses + for name in loss_names: + if name in self.keras_model.metrics_names: + continue + layer = self.keras_model.get_layer(name) + self.keras_model.metrics_names.append(name) + loss = ( + tf.reduce_mean(layer.output, keepdims=True) + * self.config.LOSS_WEIGHTS.get(name, 1.)) + self.keras_model.metrics_tensors.append(loss) + + def set_trainable(self, layer_regex, keras_model=None, indent=0, verbose=1): + """Sets model layers as trainable if their names match + the given regular expression. + """ + # Print message on the first call (but not on recursive calls) + if verbose > 0 and keras_model is None: + log("Selecting layers to train") + + keras_model = keras_model or self.keras_model + + # In multi-GPU training, we wrap the model. Get layers + # of the inner model because they have the weights. + layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\ + else keras_model.layers + + for layer in layers: + # Is the layer a model? + if layer.__class__.__name__ == 'Model': + print("In model: ", layer.name) + self.set_trainable( + layer_regex, keras_model=layer, indent=indent + 4) + continue + + if not layer.weights: + continue + # Is it trainable? + trainable = bool(re.fullmatch(layer_regex, layer.name)) + # Update layer. If layer is a container, update inner layer. + if layer.__class__.__name__ == 'TimeDistributed': + layer.layer.trainable = trainable + else: + layer.trainable = trainable + # Print trainable layer names + if trainable and verbose > 0: + log("{}{:20} ({})".format(" " * indent, layer.name, + layer.__class__.__name__)) + + def set_log_dir(self, model_path=None): + """Sets the model log directory and epoch counter. + + model_path: If None, or a format different from what this code uses + then set a new log directory and start epochs from 0. Otherwise, + extract the log directory and the epoch counter from the file + name. + """ + # Set date and epoch counter as if starting a new model + self.epoch = 0 + now = datetime.datetime.now() + + # If we have a model path with date and epochs use them + if model_path: + # Continue from we left of. Get epoch and date from the file name + # A sample model path might look like: + # \path\to\logs\coco20171029T2315\mask_rcnn_coco_0001.h5 (Windows) + # /path/to/logs/coco20171029T2315/mask_rcnn_coco_0001.h5 (Linux) + regex = r".*[/\\][\w-]+(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})[/\\]mask\_rcnn\_[\w-]+(\d{4})\.h5" + m = re.match(regex, model_path) + if m: + now = datetime.datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), + int(m.group(4)), int(m.group(5))) + # Epoch number in file is 1-based, and in Keras code it's 0-based. + # So, adjust for that then increment by one to start from the next epoch + self.epoch = int(m.group(6)) - 1 + 1 + print('Re-starting from epoch %d' % self.epoch) + + # Directory for training logs + self.log_dir = os.path.join(self.model_dir, "{}{:%Y%m%dT%H%M}".format( + self.config.NAME.lower(), now)) + + # Path to save after each epoch. Include placeholders that get filled by Keras. + self.checkpoint_path = os.path.join(self.log_dir, "mask_rcnn_{}_*epoch*.h5".format( + self.config.NAME.lower())) + self.checkpoint_path = self.checkpoint_path.replace( + "*epoch*", "{epoch:04d}") + + def train(self, train_dataset, val_dataset, learning_rate, epochs, layers, + augmentation=None, custom_callbacks=None, no_augmentation_sources=None): + """Train the model. + train_dataset, val_dataset: Training and validation Dataset objects. + learning_rate: The learning rate to train with + epochs: Number of training epochs. Note that previous training epochs + are considered to be done alreay, so this actually determines + the epochs to train in total rather than in this particaular + call. + layers: Allows selecting wich layers to train. It can be: + - A regular expression to match layer names to train + - One of these predefined values: + heads: The RPN, classifier and mask heads of the network + all: All the layers + 3+: Train Resnet stage 3 and up + 4+: Train Resnet stage 4 and up + 5+: Train Resnet stage 5 and up + augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) + augmentation. For example, passing imgaug.augmenters.Fliplr(0.5) + flips images right/left 50% of the time. You can pass complex + augmentations as well. This augmentation applies 50% of the + time, and when it does it flips images right/left half the time + and adds a Gaussian blur with a random sigma in range 0 to 5. + + augmentation = imgaug.augmenters.Sometimes(0.5, [ + imgaug.augmenters.Fliplr(0.5), + imgaug.augmenters.GaussianBlur(sigma=(0.0, 5.0)) + ]) + custom_callbacks: Optional. Add custom callbacks to be called + with the keras fit_generator method. Must be list of type keras.callbacks. + no_augmentation_sources: Optional. List of sources to exclude for + augmentation. A source is string that identifies a dataset and is + defined in the Dataset class. + """ + assert self.mode == "training", "Create model in training mode." + + # Pre-defined layer regular expressions + layer_regex = { + # all layers but the backbone + "heads": r"(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)", + # From a specific Resnet stage and up + "3+": r"(res3.*)|(bn3.*)|(res4.*)|(bn4.*)|(res5.*)|(bn5.*)|(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)", + "4+": r"(res4.*)|(bn4.*)|(res5.*)|(bn5.*)|(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)", + "5+": r"(res5.*)|(bn5.*)|(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)", + # All layers + "all": ".*", + } + if layers in layer_regex.keys(): + layers = layer_regex[layers] + + # Data generators + train_generator = data_generator(train_dataset, self.config, shuffle=True, + augmentation=augmentation, + batch_size=self.config.BATCH_SIZE, + no_augmentation_sources=no_augmentation_sources) + val_generator = data_generator(val_dataset, self.config, shuffle=True, + batch_size=self.config.BATCH_SIZE) + + # Create log_dir if it does not exist + if not os.path.exists(self.log_dir): + os.makedirs(self.log_dir) + + # Callbacks + callbacks = [ + keras.callbacks.TensorBoard(log_dir=self.log_dir, + histogram_freq=0, write_graph=True, write_images=False), + keras.callbacks.ModelCheckpoint(self.checkpoint_path, + verbose=0, save_weights_only=True), + ] + + # Add custom callbacks to the list + if custom_callbacks: + callbacks += custom_callbacks + + # Train + log("\nStarting at epoch {}. LR={}\n".format(self.epoch, learning_rate)) + log("Checkpoint Path: {}".format(self.checkpoint_path)) + self.set_trainable(layers) + self.compile(learning_rate, self.config.LEARNING_MOMENTUM) + + # Work-around for Windows: Keras fails on Windows when using + # multiprocessing workers. See discussion here: + # https://github.com/matterport/Mask_RCNN/issues/13#issuecomment-353124009 + if os.name is 'nt': + workers = 0 + else: + workers = multiprocessing.cpu_count() + + self.keras_model.fit_generator( + train_generator, + initial_epoch=self.epoch, + epochs=epochs, + steps_per_epoch=self.config.STEPS_PER_EPOCH, + callbacks=callbacks, + validation_data=val_generator, + validation_steps=self.config.VALIDATION_STEPS, + max_queue_size=50, + workers=workers, + use_multiprocessing=True, + ) + self.epoch = max(self.epoch, epochs) + + def mold_inputs(self, images): + """Takes a list of images and modifies them to the format expected + as an input to the neural network. + images: List of image matrices [height,width,depth]. Images can have + different sizes. + + Returns 3 Numpy matrices: + molded_images: [N, h, w, 3]. Images resized and normalized. + image_metas: [N, length of meta data]. Details about each image. + windows: [N, (y1, x1, y2, x2)]. The portion of the image that has the + original image (padding excluded). + """ + molded_images = [] + image_metas = [] + windows = [] + for image in images: + # Resize image + # TODO: move resizing to mold_image() + molded_image, window, scale, padding, crop = utils.resize_image( + image, + min_dim=self.config.IMAGE_MIN_DIM, + min_scale=self.config.IMAGE_MIN_SCALE, + max_dim=self.config.IMAGE_MAX_DIM, + mode=self.config.IMAGE_RESIZE_MODE) + molded_image = mold_image(molded_image, self.config) + # Build image_meta + image_meta = compose_image_meta( + 0, image.shape, molded_image.shape, window, scale, + np.zeros([self.config.NUM_CLASSES], dtype=np.int32)) + # Append + molded_images.append(molded_image) + windows.append(window) + image_metas.append(image_meta) + # Pack into arrays + molded_images = np.stack(molded_images) + image_metas = np.stack(image_metas) + windows = np.stack(windows) + return molded_images, image_metas, windows + + def unmold_detections(self, detections, mrcnn_mask, original_image_shape, + image_shape, window): + """Reformats the detections of one image from the format of the neural + network output to a format suitable for use in the rest of the + application. + + detections: [N, (y1, x1, y2, x2, class_id, score)] in normalized coordinates + mrcnn_mask: [N, height, width, num_classes] + original_image_shape: [H, W, C] Original image shape before resizing + image_shape: [H, W, C] Shape of the image after resizing and padding + window: [y1, x1, y2, x2] Pixel coordinates of box in the image where the real + image is excluding the padding. + + Returns: + boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels + class_ids: [N] Integer class IDs for each bounding box + scores: [N] Float probability scores of the class_id + masks: [height, width, num_instances] Instance masks + """ + # How many detections do we have? + # Detections array is padded with zeros. Find the first class_id == 0. + zero_ix = np.where(detections[:, 4] == 0)[0] + N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0] + + # Extract boxes, class_ids, scores, and class-specific masks + boxes = detections[:N, :4] + class_ids = detections[:N, 4].astype(np.int32) + scores = detections[:N, 5] + masks = mrcnn_mask[np.arange(N), :, :, class_ids] + + # Translate normalized coordinates in the resized image to pixel + # coordinates in the original image before resizing + window = utils.norm_boxes(window, image_shape[:2]) + wy1, wx1, wy2, wx2 = window + shift = np.array([wy1, wx1, wy1, wx1]) + wh = wy2 - wy1 # window height + ww = wx2 - wx1 # window width + scale = np.array([wh, ww, wh, ww]) + # Convert boxes to normalized coordinates on the window + boxes = np.divide(boxes - shift, scale) + # Convert boxes to pixel coordinates on the original image + boxes = utils.denorm_boxes(boxes, original_image_shape[:2]) + + # Filter out detections with zero area. Happens in early training when + # network weights are still random + exclude_ix = np.where( + (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0] + if exclude_ix.shape[0] > 0: + boxes = np.delete(boxes, exclude_ix, axis=0) + class_ids = np.delete(class_ids, exclude_ix, axis=0) + scores = np.delete(scores, exclude_ix, axis=0) + masks = np.delete(masks, exclude_ix, axis=0) + N = class_ids.shape[0] + + # Resize masks to original image size and set boundary threshold. + full_masks = [] + for i in range(N): + # Convert neural network mask to full size mask + full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape) + full_masks.append(full_mask) + full_masks = np.stack(full_masks, axis=-1)\ + if full_masks else np.empty(original_image_shape[:2] + (0,)) + + return boxes, class_ids, scores, full_masks + + def detect(self, images, verbose=0): + """Runs the detection pipeline. + + images: List of images, potentially of different sizes. + + Returns a list of dicts, one dict per image. The dict contains: + rois: [N, (y1, x1, y2, x2)] detection bounding boxes + class_ids: [N] int class IDs + scores: [N] float probability scores for the class IDs + masks: [H, W, N] instance binary masks + """ + assert self.mode == "inference", "Create model in inference mode." + assert len( + images) == self.config.BATCH_SIZE, "len(images) must be equal to BATCH_SIZE" + + if verbose: + log("Processing {} images".format(len(images))) + for image in images: + log("image", image) + + # Mold inputs to format expected by the neural network + molded_images, image_metas, windows = self.mold_inputs(images) + + # Validate image sizes + # All images in a batch MUST be of the same size + image_shape = molded_images[0].shape + for g in molded_images[1:]: + assert g.shape == image_shape,\ + "After resizing, all images must have the same size. Check IMAGE_RESIZE_MODE and image sizes." + + # Anchors + anchors = self.get_anchors(image_shape) + # Duplicate across the batch dimension because Keras requires it + # TODO: can this be optimized to avoid duplicating the anchors? + anchors = np.broadcast_to(anchors, (self.config.BATCH_SIZE,) + anchors.shape) + + if verbose: + log("molded_images", molded_images) + log("image_metas", image_metas) + log("anchors", anchors) + # Run object detection + detections, _, _, mrcnn_mask, _, _, _ =\ + self.keras_model.predict([molded_images, image_metas, anchors], verbose=0) + # Process detections + results = [] + for i, image in enumerate(images): + final_rois, final_class_ids, final_scores, final_masks =\ + self.unmold_detections(detections[i], mrcnn_mask[i], + image.shape, molded_images[i].shape, + windows[i]) + results.append({ + "rois": final_rois, + "class_ids": final_class_ids, + "scores": final_scores, + "masks": final_masks, + }) + return results + + def detect_molded(self, molded_images, image_metas, verbose=0): + """Runs the detection pipeline, but expect inputs that are + molded already. Used mostly for debugging and inspecting + the model. + + molded_images: List of images loaded using load_image_gt() + image_metas: image meta data, also returned by load_image_gt() + + Returns a list of dicts, one dict per image. The dict contains: + rois: [N, (y1, x1, y2, x2)] detection bounding boxes + class_ids: [N] int class IDs + scores: [N] float probability scores for the class IDs + masks: [H, W, N] instance binary masks + """ + assert self.mode == "inference", "Create model in inference mode." + assert len(molded_images) == self.config.BATCH_SIZE,\ + "Number of images must be equal to BATCH_SIZE" + + if verbose: + log("Processing {} images".format(len(molded_images))) + for image in molded_images: + log("image", image) + + # Validate image sizes + # All images in a batch MUST be of the same size + image_shape = molded_images[0].shape + for g in molded_images[1:]: + assert g.shape == image_shape, "Images must have the same size" + + # Anchors + anchors = self.get_anchors(image_shape) + # Duplicate across the batch dimension because Keras requires it + # TODO: can this be optimized to avoid duplicating the anchors? + anchors = np.broadcast_to(anchors, (self.config.BATCH_SIZE,) + anchors.shape) + + if verbose: + log("molded_images", molded_images) + log("image_metas", image_metas) + log("anchors", anchors) + # Run object detection + detections, _, _, mrcnn_mask, _, _, _ =\ + self.keras_model.predict([molded_images, image_metas, anchors], verbose=0) + # Process detections + results = [] + for i, image in enumerate(molded_images): + window = [0, 0, image.shape[0], image.shape[1]] + final_rois, final_class_ids, final_scores, final_masks =\ + self.unmold_detections(detections[i], mrcnn_mask[i], + image.shape, molded_images[i].shape, + window) + results.append({ + "rois": final_rois, + "class_ids": final_class_ids, + "scores": final_scores, + "masks": final_masks, + }) + return results + + def get_anchors(self, image_shape): + """Returns anchor pyramid for the given image size.""" + backbone_shapes = compute_backbone_shapes(self.config, image_shape) + # Cache anchors and reuse if image shape is the same + if not hasattr(self, "_anchor_cache"): + self._anchor_cache = {} + if not tuple(image_shape) in self._anchor_cache: + # Generate Anchors + a = utils.generate_pyramid_anchors( + self.config.RPN_ANCHOR_SCALES, + self.config.RPN_ANCHOR_RATIOS, + backbone_shapes, + self.config.BACKBONE_STRIDES, + self.config.RPN_ANCHOR_STRIDE) + # Keep a copy of the latest anchors in pixel coordinates because + # it's used in inspect_model notebooks. + # TODO: Remove this after the notebook are refactored to not use it + self.anchors = a + # Normalize coordinates + self._anchor_cache[tuple(image_shape)] = utils.norm_boxes(a, image_shape[:2]) + return self._anchor_cache[tuple(image_shape)] + + def ancestor(self, tensor, name, checked=None): + """Finds the ancestor of a TF tensor in the computation graph. + tensor: TensorFlow symbolic tensor. + name: Name of ancestor tensor to find + checked: For internal use. A list of tensors that were already + searched to avoid loops in traversing the graph. + """ + checked = checked if checked is not None else [] + # Put a limit on how deep we go to avoid very long loops + if len(checked) > 500: + return None + # Convert name to a regex and allow matching a number prefix + # because Keras adds them automatically + if isinstance(name, str): + name = re.compile(name.replace("/", r"(\_\d+)*/")) + + parents = tensor.op.inputs + for p in parents: + if p in checked: + continue + if bool(re.fullmatch(name, p.name)): + return p + checked.append(p) + a = self.ancestor(p, name, checked) + if a is not None: + return a + return None + + def find_trainable_layer(self, layer): + """If a layer is encapsulated by another layer, this function + digs through the encapsulation and returns the layer that holds + the weights. + """ + if layer.__class__.__name__ == 'TimeDistributed': + return self.find_trainable_layer(layer.layer) + return layer + + def get_trainable_layers(self): + """Returns a list of layers that have weights.""" + layers = [] + # Loop through all layers + for l in self.keras_model.layers: + # If layer is a wrapper, find inner trainable layer + l = self.find_trainable_layer(l) + # Include layer if it has weights + if l.get_weights(): + layers.append(l) + return layers + + def run_graph(self, images, outputs, image_metas=None): + """Runs a sub-set of the computation graph that computes the given + outputs. + + image_metas: If provided, the images are assumed to be already + molded (i.e. resized, padded, and normalized) + + outputs: List of tuples (name, tensor) to compute. The tensors are + symbolic TensorFlow tensors and the names are for easy tracking. + + Returns an ordered dict of results. Keys are the names received in the + input and values are Numpy arrays. + """ + model = self.keras_model + + # Organize desired outputs into an ordered dict + outputs = OrderedDict(outputs) + for o in outputs.values(): + assert o is not None + + # Build a Keras function to run parts of the computation graph + inputs = model.inputs + if model.uses_learning_phase and not isinstance(K.learning_phase(), int): + inputs += [K.learning_phase()] + kf = K.function(model.inputs, list(outputs.values())) + + # Prepare inputs + if image_metas is None: + molded_images, image_metas, _ = self.mold_inputs(images) + else: + molded_images = images + image_shape = molded_images[0].shape + # Anchors + anchors = self.get_anchors(image_shape) + # Duplicate across the batch dimension because Keras requires it + # TODO: can this be optimized to avoid duplicating the anchors? + anchors = np.broadcast_to(anchors, (self.config.BATCH_SIZE,) + anchors.shape) + model_in = [molded_images, image_metas, anchors] + + # Run inference + if model.uses_learning_phase and not isinstance(K.learning_phase(), int): + model_in.append(0.) + outputs_np = kf(model_in) + + # Pack the generated Numpy arrays into a a dict and log the results. + outputs_np = OrderedDict([(k, v) + for k, v in zip(outputs.keys(), outputs_np)]) + for k, v in outputs_np.items(): + log(k, v) + return outputs_np + + +############################################################ +# Data Formatting +############################################################ + +def compose_image_meta(image_id, original_image_shape, image_shape, + window, scale, active_class_ids): + """Takes attributes of an image and puts them in one 1D array. + + image_id: An int ID of the image. Useful for debugging. + original_image_shape: [H, W, C] before resizing or padding. + image_shape: [H, W, C] after resizing and padding + window: (y1, x1, y2, x2) in pixels. The area of the image where the real + image is (excluding the padding) + scale: The scaling factor applied to the original image (float32) + active_class_ids: List of class_ids available in the dataset from which + the image came. Useful if training on images from multiple datasets + where not all classes are present in all datasets. + """ + meta = np.array( + [image_id] + # size=1 + list(original_image_shape) + # size=3 + list(image_shape) + # size=3 + list(window) + # size=4 (y1, x1, y2, x2) in image cooredinates + [scale] + # size=1 + list(active_class_ids) # size=num_classes + ) + return meta + + +def parse_image_meta(meta): + """Parses an array that contains image attributes to its components. + See compose_image_meta() for more details. + + meta: [batch, meta length] where meta length depends on NUM_CLASSES + + Returns a dict of the parsed values. + """ + image_id = meta[:, 0] + original_image_shape = meta[:, 1:4] + image_shape = meta[:, 4:7] + window = meta[:, 7:11] # (y1, x1, y2, x2) window of image in in pixels + scale = meta[:, 11] + active_class_ids = meta[:, 12:] + return { + "image_id": image_id.astype(np.int32), + "original_image_shape": original_image_shape.astype(np.int32), + "image_shape": image_shape.astype(np.int32), + "window": window.astype(np.int32), + "scale": scale.astype(np.float32), + "active_class_ids": active_class_ids.astype(np.int32), + } + + +def parse_image_meta_graph(meta): + """Parses a tensor that contains image attributes to its components. + See compose_image_meta() for more details. + + meta: [batch, meta length] where meta length depends on NUM_CLASSES + + Returns a dict of the parsed tensors. + """ + image_id = meta[:, 0] + original_image_shape = meta[:, 1:4] + image_shape = meta[:, 4:7] + window = meta[:, 7:11] # (y1, x1, y2, x2) window of image in in pixels + scale = meta[:, 11] + active_class_ids = meta[:, 12:] + return { + "image_id": image_id, + "original_image_shape": original_image_shape, + "image_shape": image_shape, + "window": window, + "scale": scale, + "active_class_ids": active_class_ids, + } + + +def mold_image(images, config): + """Expects an RGB image (or array of images) and subtracts + the mean pixel and converts it to float. Expects image + colors in RGB order. + """ + return images.astype(np.float32) - config.MEAN_PIXEL + + +def unmold_image(normalized_images, config): + """Takes a image normalized with mold() and returns the original.""" + return (normalized_images + config.MEAN_PIXEL).astype(np.uint8) + + +############################################################ +# Miscellenous Graph Functions +############################################################ + +def trim_zeros_graph(boxes, name='trim_zeros'): + """Often boxes are represented with matrices of shape [N, 4] and + are padded with zeros. This removes zero boxes. + + boxes: [N, 4] matrix of boxes. + non_zeros: [N] a 1D boolean mask identifying the rows to keep + """ + non_zeros = tf.cast(tf.reduce_sum(tf.abs(boxes), axis=1), tf.bool) + boxes = tf.boolean_mask(boxes, non_zeros, name=name) + return boxes, non_zeros + + +def batch_pack_graph(x, counts, num_rows): + """Picks different number of values from each row + in x depending on the values in counts. + """ + outputs = [] + for i in range(num_rows): + outputs.append(x[i, :counts[i]]) + return tf.concat(outputs, axis=0) + + +def norm_boxes_graph(boxes, shape): + """Converts boxes from pixel coordinates to normalized coordinates. + boxes: [..., (y1, x1, y2, x2)] in pixel coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [..., (y1, x1, y2, x2)] in normalized coordinates + """ + h, w = tf.split(tf.cast(shape, tf.float32), 2) + scale = tf.concat([h, w, h, w], axis=-1) - tf.constant(1.0) + shift = tf.constant([0., 0., 1., 1.]) + return tf.divide(boxes - shift, scale) + + +def denorm_boxes_graph(boxes, shape): + """Converts boxes from normalized coordinates to pixel coordinates. + boxes: [..., (y1, x1, y2, x2)] in normalized coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [..., (y1, x1, y2, x2)] in pixel coordinates + """ + h, w = tf.split(tf.cast(shape, tf.float32), 2) + scale = tf.concat([h, w, h, w], axis=-1) - tf.constant(1.0) + shift = tf.constant([0., 0., 1., 1.]) + return tf.cast(tf.round(tf.multiply(boxes, scale) + shift), tf.int32) diff --git a/mask_rcnn/mrcnn/parallel_model.py b/mask_rcnn/mrcnn/parallel_model.py new file mode 100644 index 00000000..0a28fdf6 --- /dev/null +++ b/mask_rcnn/mrcnn/parallel_model.py @@ -0,0 +1,177 @@ +""" +Mask R-CNN +Multi-GPU Support for Keras. + +Copyright (c) 2017 Matterport, Inc. +Licensed under the MIT License (see LICENSE for details) +Written by Waleed Abdulla + +Ideas and a small code snippets from these sources: +https://github.com/fchollet/keras/issues/2436 +https://medium.com/@kuza55/transparent-multi-gpu-training-on-tensorflow-with-keras-8b0016fd9012 +https://github.com/avolkov1/keras_experiments/blob/master/keras_exp/multigpu/ +https://github.com/fchollet/keras/blob/master/keras/utils/training_utils.py +""" +import warnings +warnings.filterwarnings('ignore', category=DeprecationWarning) +warnings.filterwarnings('ignore', category=FutureWarning) +import tensorflow as tf +import keras.backend as K +import keras.layers as KL +import keras.models as KM + + +class ParallelModel(KM.Model): + """Subclasses the standard Keras Model and adds multi-GPU support. + It works by creating a copy of the model on each GPU. Then it slices + the inputs and sends a slice to each copy of the model, and then + merges the outputs together and applies the loss on the combined + outputs. + """ + + def __init__(self, keras_model, gpu_count): + """Class constructor. + keras_model: The Keras model to parallelize + gpu_count: Number of GPUs. Must be > 1 + """ + self.inner_model = keras_model + self.gpu_count = gpu_count + merged_outputs = self.make_parallel() + super(ParallelModel, self).__init__(inputs=self.inner_model.inputs, + outputs=merged_outputs) + + def __getattribute__(self, attrname): + """Redirect loading and saving methods to the inner model. That's where + the weights are stored.""" + if 'load' in attrname or 'save' in attrname: + return getattr(self.inner_model, attrname) + return super(ParallelModel, self).__getattribute__(attrname) + + def summary(self, *args, **kwargs): + """Override summary() to display summaries of both, the wrapper + and inner models.""" + super(ParallelModel, self).summary(*args, **kwargs) + self.inner_model.summary(*args, **kwargs) + + def make_parallel(self): + """Creates a new wrapper model that consists of multiple replicas of + the original model placed on different GPUs. + """ + # Slice inputs. Slice inputs on the CPU to avoid sending a copy + # of the full inputs to all GPUs. Saves on bandwidth and memory. + input_slices = {name: tf.split(x, self.gpu_count) + for name, x in zip(self.inner_model.input_names, + self.inner_model.inputs)} + + output_names = self.inner_model.output_names + outputs_all = [] + for i in range(len(self.inner_model.outputs)): + outputs_all.append([]) + + # Run the model call() on each GPU to place the ops there + for i in range(self.gpu_count): + with tf.device('/gpu:%d' % i): + with tf.name_scope('tower_%d' % i): + # Run a slice of inputs through this replica + zipped_inputs = zip(self.inner_model.input_names, + self.inner_model.inputs) + inputs = [ + KL.Lambda(lambda s: input_slices[name][i], + output_shape=lambda s: (None,) + s[1:])(tensor) + for name, tensor in zipped_inputs] + # Create the model replica and get the outputs + outputs = self.inner_model(inputs) + if not isinstance(outputs, list): + outputs = [outputs] + # Save the outputs for merging back together later + for l, o in enumerate(outputs): + outputs_all[l].append(o) + + # Merge outputs on CPU + with tf.device('/cpu:0'): + merged = [] + for outputs, name in zip(outputs_all, output_names): + # Concatenate or average outputs? + # Outputs usually have a batch dimension and we concatenate + # across it. If they don't, then the output is likely a loss + # or a metric value that gets averaged across the batch. + # Keras expects losses and metrics to be scalars. + if K.int_shape(outputs[0]) == (): + # Average + m = KL.Lambda(lambda o: tf.add_n(o) / len(outputs), name=name)(outputs) + else: + # Concatenate + m = KL.Concatenate(axis=0, name=name)(outputs) + merged.append(m) + return merged + + +if __name__ == "__main__": + # Testing code below. It creates a simple model to train on MNIST and + # tries to run it on 2 GPUs. It saves the graph so it can be viewed + # in TensorBoard. Run it as: + # + # python3 parallel_model.py + + import os + import numpy as np + import keras.optimizers + from keras.datasets import mnist + from keras.preprocessing.image import ImageDataGenerator + + GPU_COUNT = 2 + + # Root directory of the project + ROOT_DIR = os.path.abspath("../") + + # Directory to save logs and trained model + MODEL_DIR = os.path.join(ROOT_DIR, "logs") + + def build_model(x_train, num_classes): + # Reset default graph. Keras leaves old ops in the graph, + # which are ignored for execution but clutter graph + # visualization in TensorBoard. + tf.reset_default_graph() + + inputs = KL.Input(shape=x_train.shape[1:], name="input_image") + x = KL.Conv2D(32, (3, 3), activation='relu', padding="same", + name="conv1")(inputs) + x = KL.Conv2D(64, (3, 3), activation='relu', padding="same", + name="conv2")(x) + x = KL.MaxPooling2D(pool_size=(2, 2), name="pool1")(x) + x = KL.Flatten(name="flat1")(x) + x = KL.Dense(128, activation='relu', name="dense1")(x) + x = KL.Dense(num_classes, activation='softmax', name="dense2")(x) + + return KM.Model(inputs, x, "digit_classifier_model") + + # Load MNIST Data + (x_train, y_train), (x_test, y_test) = mnist.load_data() + x_train = np.expand_dims(x_train, -1).astype('float32') / 255 + x_test = np.expand_dims(x_test, -1).astype('float32') / 255 + + print('x_train shape:', x_train.shape) + print('x_test shape:', x_test.shape) + + # Build data generator and model + datagen = ImageDataGenerator() + model = build_model(x_train, 10) + + # Add multi-GPU support. + model = ParallelModel(model, GPU_COUNT) + + optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, clipnorm=5.0) + + model.compile(loss='sparse_categorical_crossentropy', + optimizer=optimizer, metrics=['accuracy']) + + model.summary() + + # Train + model.fit_generator( + datagen.flow(x_train, y_train, batch_size=64), + steps_per_epoch=50, epochs=10, verbose=1, + validation_data=(x_test, y_test), + callbacks=[keras.callbacks.TensorBoard(log_dir=MODEL_DIR, + write_graph=True)] + ) diff --git a/mask_rcnn/mrcnn/utils.py b/mask_rcnn/mrcnn/utils.py new file mode 100644 index 00000000..e0da612a --- /dev/null +++ b/mask_rcnn/mrcnn/utils.py @@ -0,0 +1,911 @@ +""" +Mask R-CNN +Common utility functions and classes. + +Copyright (c) 2017 Matterport, Inc. +Licensed under the MIT License (see LICENSE for details) +Written by Waleed Abdulla +""" + +import sys +import os +import logging +import math +import random +import numpy as np +import warnings +warnings.filterwarnings('ignore', category=DeprecationWarning) +warnings.filterwarnings('ignore', category=FutureWarning) +import tensorflow as tf +import scipy +import skimage.color +import skimage.io +import skimage.transform +import urllib.request +import shutil +import warnings +from distutils.version import LooseVersion + +# URL from which to download the latest COCO trained weights +COCO_MODEL_URL = "https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5" + + +############################################################ +# Bounding Boxes +############################################################ + +def extract_bboxes(mask): + """Compute bounding boxes from masks. + mask: [height, width, num_instances]. Mask pixels are either 1 or 0. + + Returns: bbox array [num_instances, (y1, x1, y2, x2)]. + """ + boxes = np.zeros([mask.shape[-1], 4], dtype=np.int32) + for i in range(mask.shape[-1]): + m = mask[:, :, i] + # Bounding box. + horizontal_indicies = np.where(np.any(m, axis=0))[0] + vertical_indicies = np.where(np.any(m, axis=1))[0] + if horizontal_indicies.shape[0]: + x1, x2 = horizontal_indicies[[0, -1]] + y1, y2 = vertical_indicies[[0, -1]] + # x2 and y2 should not be part of the box. Increment by 1. + x2 += 1 + y2 += 1 + else: + # No mask for this instance. Might happen due to + # resizing or cropping. Set bbox to zeros + x1, x2, y1, y2 = 0, 0, 0, 0 + boxes[i] = np.array([y1, x1, y2, x2]) + return boxes.astype(np.int32) + + +def compute_iou(box, boxes, box_area, boxes_area): + """Calculates IoU of the given box with the array of the given boxes. + box: 1D vector [y1, x1, y2, x2] + boxes: [boxes_count, (y1, x1, y2, x2)] + box_area: float. the area of 'box' + boxes_area: array of length boxes_count. + + Note: the areas are passed in rather than calculated here for + efficiency. Calculate once in the caller to avoid duplicate work. + """ + # Calculate intersection areas + y1 = np.maximum(box[0], boxes[:, 0]) + y2 = np.minimum(box[2], boxes[:, 2]) + x1 = np.maximum(box[1], boxes[:, 1]) + x2 = np.minimum(box[3], boxes[:, 3]) + intersection = np.maximum(x2 - x1, 0) * np.maximum(y2 - y1, 0) + union = box_area + boxes_area[:] - intersection[:] + iou = intersection / union + return iou + + +def compute_overlaps(boxes1, boxes2): + """Computes IoU overlaps between two sets of boxes. + boxes1, boxes2: [N, (y1, x1, y2, x2)]. + + For better performance, pass the largest set first and the smaller second. + """ + # Areas of anchors and GT boxes + area1 = (boxes1[:, 2] - boxes1[:, 0]) * (boxes1[:, 3] - boxes1[:, 1]) + area2 = (boxes2[:, 2] - boxes2[:, 0]) * (boxes2[:, 3] - boxes2[:, 1]) + + # Compute overlaps to generate matrix [boxes1 count, boxes2 count] + # Each cell contains the IoU value. + overlaps = np.zeros((boxes1.shape[0], boxes2.shape[0])) + for i in range(overlaps.shape[1]): + box2 = boxes2[i] + overlaps[:, i] = compute_iou(box2, boxes1, area2[i], area1) + return overlaps + + +def compute_overlaps_masks(masks1, masks2): + """Computes IoU overlaps between two sets of masks. + masks1, masks2: [Height, Width, instances] + """ + + # If either set of masks is empty return empty result + if masks1.shape[-1] == 0 or masks2.shape[-1] == 0: + return np.zeros((masks1.shape[-1], masks2.shape[-1])) + # flatten masks and compute their areas + masks1 = np.reshape(masks1 > .5, (-1, masks1.shape[-1])).astype(np.float32) + masks2 = np.reshape(masks2 > .5, (-1, masks2.shape[-1])).astype(np.float32) + area1 = np.sum(masks1, axis=0) + area2 = np.sum(masks2, axis=0) + + # intersections and union + intersections = np.dot(masks1.T, masks2) + union = area1[:, None] + area2[None, :] - intersections + overlaps = intersections / union + + return overlaps + + +def non_max_suppression(boxes, scores, threshold): + """Performs non-maximum suppression and returns indices of kept boxes. + boxes: [N, (y1, x1, y2, x2)]. Notice that (y2, x2) lays outside the box. + scores: 1-D array of box scores. + threshold: Float. IoU threshold to use for filtering. + """ + assert boxes.shape[0] > 0 + if boxes.dtype.kind != "f": + boxes = boxes.astype(np.float32) + + # Compute box areas + y1 = boxes[:, 0] + x1 = boxes[:, 1] + y2 = boxes[:, 2] + x2 = boxes[:, 3] + area = (y2 - y1) * (x2 - x1) + + # Get indicies of boxes sorted by scores (highest first) + ixs = scores.argsort()[::-1] + + pick = [] + while len(ixs) > 0: + # Pick top box and add its index to the list + i = ixs[0] + pick.append(i) + # Compute IoU of the picked box with the rest + iou = compute_iou(boxes[i], boxes[ixs[1:]], area[i], area[ixs[1:]]) + # Identify boxes with IoU over the threshold. This + # returns indices into ixs[1:], so add 1 to get + # indices into ixs. + remove_ixs = np.where(iou > threshold)[0] + 1 + # Remove indices of the picked and overlapped boxes. + ixs = np.delete(ixs, remove_ixs) + ixs = np.delete(ixs, 0) + return np.array(pick, dtype=np.int32) + + +def apply_box_deltas(boxes, deltas): + """Applies the given deltas to the given boxes. + boxes: [N, (y1, x1, y2, x2)]. Note that (y2, x2) is outside the box. + deltas: [N, (dy, dx, log(dh), log(dw))] + """ + boxes = boxes.astype(np.float32) + # Convert to y, x, h, w + height = boxes[:, 2] - boxes[:, 0] + width = boxes[:, 3] - boxes[:, 1] + center_y = boxes[:, 0] + 0.5 * height + center_x = boxes[:, 1] + 0.5 * width + # Apply deltas + center_y += deltas[:, 0] * height + center_x += deltas[:, 1] * width + height *= np.exp(deltas[:, 2]) + width *= np.exp(deltas[:, 3]) + # Convert back to y1, x1, y2, x2 + y1 = center_y - 0.5 * height + x1 = center_x - 0.5 * width + y2 = y1 + height + x2 = x1 + width + return np.stack([y1, x1, y2, x2], axis=1) + + +def box_refinement_graph(box, gt_box): + """Compute refinement needed to transform box to gt_box. + box and gt_box are [N, (y1, x1, y2, x2)] + """ + box = tf.cast(box, tf.float32) + gt_box = tf.cast(gt_box, tf.float32) + + height = box[:, 2] - box[:, 0] + width = box[:, 3] - box[:, 1] + center_y = box[:, 0] + 0.5 * height + center_x = box[:, 1] + 0.5 * width + + gt_height = gt_box[:, 2] - gt_box[:, 0] + gt_width = gt_box[:, 3] - gt_box[:, 1] + gt_center_y = gt_box[:, 0] + 0.5 * gt_height + gt_center_x = gt_box[:, 1] + 0.5 * gt_width + + dy = (gt_center_y - center_y) / height + dx = (gt_center_x - center_x) / width + dh = tf.log(gt_height / height) + dw = tf.log(gt_width / width) + + result = tf.stack([dy, dx, dh, dw], axis=1) + return result + + +def box_refinement(box, gt_box): + """Compute refinement needed to transform box to gt_box. + box and gt_box are [N, (y1, x1, y2, x2)]. (y2, x2) is + assumed to be outside the box. + """ + box = box.astype(np.float32) + gt_box = gt_box.astype(np.float32) + + height = box[:, 2] - box[:, 0] + width = box[:, 3] - box[:, 1] + center_y = box[:, 0] + 0.5 * height + center_x = box[:, 1] + 0.5 * width + + gt_height = gt_box[:, 2] - gt_box[:, 0] + gt_width = gt_box[:, 3] - gt_box[:, 1] + gt_center_y = gt_box[:, 0] + 0.5 * gt_height + gt_center_x = gt_box[:, 1] + 0.5 * gt_width + + dy = (gt_center_y - center_y) / height + dx = (gt_center_x - center_x) / width + dh = np.log(gt_height / height) + dw = np.log(gt_width / width) + + return np.stack([dy, dx, dh, dw], axis=1) + + +############################################################ +# Dataset +############################################################ + +class Dataset(object): + """The base class for dataset classes. + To use it, create a new class that adds functions specific to the dataset + you want to use. For example: + + class CatsAndDogsDataset(Dataset): + def load_cats_and_dogs(self): + ... + def load_mask(self, image_id): + ... + def image_reference(self, image_id): + ... + + See COCODataset and ShapesDataset as examples. + """ + + def __init__(self, class_map=None): + self._image_ids = [] + self.image_info = [] + # Background is always the first class + self.class_info = [{"source": "", "id": 0, "name": "BG"}] + self.source_class_ids = {} + + def add_class(self, source, class_id, class_name): + assert "." not in source, "Source name cannot contain a dot" + # Does the class exist already? + for info in self.class_info: + if info['source'] == source and info["id"] == class_id: + # source.class_id combination already available, skip + return + # Add the class + self.class_info.append({ + "source": source, + "id": class_id, + "name": class_name, + }) + + def add_image(self, source, image_id, path, **kwargs): + image_info = { + "id": image_id, + "source": source, + "path": path, + } + image_info.update(kwargs) + self.image_info.append(image_info) + + def image_reference(self, image_id): + """Return a link to the image in its source Website or details about + the image that help looking it up or debugging it. + + Override for your dataset, but pass to this function + if you encounter images not in your dataset. + """ + return "" + + def prepare(self, class_map=None): + """Prepares the Dataset class for use. + + TODO: class map is not supported yet. When done, it should handle mapping + classes from different datasets to the same class ID. + """ + + def clean_name(name): + """Returns a shorter version of object names for cleaner display.""" + return ",".join(name.split(",")[:1]) + + # Build (or rebuild) everything else from the info dicts. + self.num_classes = len(self.class_info) + self.class_ids = np.arange(self.num_classes) + self.class_names = [clean_name(c["name"]) for c in self.class_info] + self.num_images = len(self.image_info) + self._image_ids = np.arange(self.num_images) + + # Mapping from source class and image IDs to internal IDs + self.class_from_source_map = {"{}.{}".format(info['source'], info['id']): id + for info, id in zip(self.class_info, self.class_ids)} + self.image_from_source_map = {"{}.{}".format(info['source'], info['id']): id + for info, id in zip(self.image_info, self.image_ids)} + + # Map sources to class_ids they support + self.sources = list(set([i['source'] for i in self.class_info])) + self.source_class_ids = {} + # Loop over datasets + for source in self.sources: + self.source_class_ids[source] = [] + # Find classes that belong to this dataset + for i, info in enumerate(self.class_info): + # Include BG class in all datasets + if i == 0 or source == info['source']: + self.source_class_ids[source].append(i) + + def map_source_class_id(self, source_class_id): + """Takes a source class ID and returns the int class ID assigned to it. + + For example: + dataset.map_source_class_id("coco.12") -> 23 + """ + return self.class_from_source_map[source_class_id] + + def get_source_class_id(self, class_id, source): + """Map an internal class ID to the corresponding class ID in the source dataset.""" + info = self.class_info[class_id] + assert info['source'] == source + return info['id'] + + @property + def image_ids(self): + return self._image_ids + + def source_image_link(self, image_id): + """Returns the path or URL to the image. + Override this to return a URL to the image if it's available online for easy + debugging. + """ + return self.image_info[image_id]["path"] + + def load_image(self, image_id): + """Load the specified image and return a [H,W,3] Numpy array. + """ + # Load image + image = skimage.io.imread(self.image_info[image_id]['path']) + # If grayscale. Convert to RGB for consistency. + if image.ndim != 3: + image = skimage.color.gray2rgb(image) + # If has an alpha channel, remove it for consistency + if image.shape[-1] == 4: + image = image[..., :3] + return image + + def load_mask(self, image_id): + """Load instance masks for the given image. + + Different datasets use different ways to store masks. Override this + method to load instance masks and return them in the form of am + array of binary masks of shape [height, width, instances]. + + Returns: + masks: A bool array of shape [height, width, instance count] with + a binary mask per instance. + class_ids: a 1D array of class IDs of the instance masks. + """ + # Override this function to load a mask from your dataset. + # Otherwise, it returns an empty mask. + logging.warning("You are using the default load_mask(), maybe you need to define your own one.") + mask = np.empty([0, 0, 0]) + class_ids = np.empty([0], np.int32) + return mask, class_ids + + +def resize_image(image, min_dim=None, max_dim=None, min_scale=None, mode="square"): + """Resizes an image keeping the aspect ratio unchanged. + + min_dim: if provided, resizes the image such that it's smaller + dimension == min_dim + max_dim: if provided, ensures that the image longest side doesn't + exceed this value. + min_scale: if provided, ensure that the image is scaled up by at least + this percent even if min_dim doesn't require it. + mode: Resizing mode. + none: No resizing. Return the image unchanged. + square: Resize and pad with zeros to get a square image + of size [max_dim, max_dim]. + pad64: Pads width and height with zeros to make them multiples of 64. + If min_dim or min_scale are provided, it scales the image up + before padding. max_dim is ignored in this mode. + The multiple of 64 is needed to ensure smooth scaling of feature + maps up and down the 6 levels of the FPN pyramid (2**6=64). + crop: Picks random crops from the image. First, scales the image based + on min_dim and min_scale, then picks a random crop of + size min_dim x min_dim. Can be used in training only. + max_dim is not used in this mode. + + Returns: + image: the resized image + window: (y1, x1, y2, x2). If max_dim is provided, padding might + be inserted in the returned image. If so, this window is the + coordinates of the image part of the full image (excluding + the padding). The x2, y2 pixels are not included. + scale: The scale factor used to resize the image + padding: Padding added to the image [(top, bottom), (left, right), (0, 0)] + """ + # Keep track of image dtype and return results in the same dtype + image_dtype = image.dtype + # Default window (y1, x1, y2, x2) and default scale == 1. + h, w = image.shape[:2] + window = (0, 0, h, w) + scale = 1 + padding = [(0, 0), (0, 0), (0, 0)] + crop = None + + if mode == "none": + return image, window, scale, padding, crop + + # Scale? + if min_dim: + # Scale up but not down + scale = max(1, min_dim / min(h, w)) + if min_scale and scale < min_scale: + scale = min_scale + + # Does it exceed max dim? + if max_dim and mode == "square": + image_max = max(h, w) + if round(image_max * scale) > max_dim: + scale = max_dim / image_max + + # Resize image using bilinear interpolation + if scale != 1: + image = resize(image, (round(h * scale), round(w * scale)), + preserve_range=True) + + # Need padding or cropping? + if mode == "square": + # Get new height and width + h, w = image.shape[:2] + top_pad = (max_dim - h) // 2 + bottom_pad = max_dim - h - top_pad + left_pad = (max_dim - w) // 2 + right_pad = max_dim - w - left_pad + padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)] + image = np.pad(image, padding, mode='constant', constant_values=0) + window = (top_pad, left_pad, h + top_pad, w + left_pad) + elif mode == "pad64": + h, w = image.shape[:2] + # Both sides must be divisible by 64 + assert min_dim % 64 == 0, "Minimum dimension must be a multiple of 64" + # Height + if h % 64 > 0: + max_h = h - (h % 64) + 64 + top_pad = (max_h - h) // 2 + bottom_pad = max_h - h - top_pad + else: + top_pad = bottom_pad = 0 + # Width + if w % 64 > 0: + max_w = w - (w % 64) + 64 + left_pad = (max_w - w) // 2 + right_pad = max_w - w - left_pad + else: + left_pad = right_pad = 0 + padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)] + image = np.pad(image, padding, mode='constant', constant_values=0) + window = (top_pad, left_pad, h + top_pad, w + left_pad) + elif mode == "crop": + # Pick a random crop + h, w = image.shape[:2] + y = random.randint(0, (h - min_dim)) + x = random.randint(0, (w - min_dim)) + crop = (y, x, min_dim, min_dim) + image = image[y:y + min_dim, x:x + min_dim] + window = (0, 0, min_dim, min_dim) + else: + raise Exception("Mode {} not supported".format(mode)) + return image.astype(image_dtype), window, scale, padding, crop + + +def resize_mask(mask, scale, padding, crop=None): + """Resizes a mask using the given scale and padding. + Typically, you get the scale and padding from resize_image() to + ensure both, the image and the mask, are resized consistently. + + scale: mask scaling factor + padding: Padding to add to the mask in the form + [(top, bottom), (left, right), (0, 0)] + """ + # Suppress warning from scipy 0.13.0, the output shape of zoom() is + # calculated with round() instead of int() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mask = scipy.ndimage.zoom(mask, zoom=[scale, scale, 1], order=0) + if crop is not None: + y, x, h, w = crop + mask = mask[y:y + h, x:x + w] + else: + mask = np.pad(mask, padding, mode='constant', constant_values=0) + return mask + + +def minimize_mask(bbox, mask, mini_shape): + """Resize masks to a smaller version to reduce memory load. + Mini-masks can be resized back to image scale using expand_masks() + + See inspect_data.ipynb notebook for more details. + """ + mini_mask = np.zeros(mini_shape + (mask.shape[-1],), dtype=bool) + for i in range(mask.shape[-1]): + # Pick slice and cast to bool in case load_mask() returned wrong dtype + m = mask[:, :, i].astype(bool) + y1, x1, y2, x2 = bbox[i][:4] + m = m[y1:y2, x1:x2] + if m.size == 0: + raise Exception("Invalid bounding box with area of zero") + # Resize with bilinear interpolation + m = resize(m, mini_shape) + mini_mask[:, :, i] = np.around(m).astype(np.bool) + return mini_mask + + +def expand_mask(bbox, mini_mask, image_shape): + """Resizes mini masks back to image size. Reverses the change + of minimize_mask(). + + See inspect_data.ipynb notebook for more details. + """ + mask = np.zeros(image_shape[:2] + (mini_mask.shape[-1],), dtype=bool) + for i in range(mask.shape[-1]): + m = mini_mask[:, :, i] + y1, x1, y2, x2 = bbox[i][:4] + h = y2 - y1 + w = x2 - x1 + # Resize with bilinear interpolation + m = resize(m, (h, w)) + mask[y1:y2, x1:x2, i] = np.around(m).astype(np.bool) + return mask + + +# TODO: Build and use this function to reduce code duplication +def mold_mask(mask, config): + pass + + +def unmold_mask(mask, bbox, image_shape): + """Converts a mask generated by the neural network to a format similar + to its original shape. + mask: [height, width] of type float. A small, typically 28x28 mask. + bbox: [y1, x1, y2, x2]. The box to fit the mask in. + + Returns a binary mask with the same size as the original image. + """ + threshold = 0.5 + y1, x1, y2, x2 = bbox + mask = resize(mask, (y2 - y1, x2 - x1)) + mask = np.where(mask >= threshold, 1, 0).astype(np.bool) + + # Put the mask in the right location. + full_mask = np.zeros(image_shape[:2], dtype=np.bool) + full_mask[y1:y2, x1:x2] = mask + return full_mask + + +############################################################ +# Anchors +############################################################ + +def generate_anchors(scales, ratios, shape, feature_stride, anchor_stride): + """ + scales: 1D array of anchor sizes in pixels. Example: [32, 64, 128] + ratios: 1D array of anchor ratios of width/height. Example: [0.5, 1, 2] + shape: [height, width] spatial shape of the feature map over which + to generate anchors. + feature_stride: Stride of the feature map relative to the image in pixels. + anchor_stride: Stride of anchors on the feature map. For example, if the + value is 2 then generate anchors for every other feature map pixel. + """ + # Get all combinations of scales and ratios + scales, ratios = np.meshgrid(np.array(scales), np.array(ratios)) + scales = scales.flatten() + ratios = ratios.flatten() + + # Enumerate heights and widths from scales and ratios + heights = scales / np.sqrt(ratios) + widths = scales * np.sqrt(ratios) + + # Enumerate shifts in feature space + shifts_y = np.arange(0, shape[0], anchor_stride) * feature_stride + shifts_x = np.arange(0, shape[1], anchor_stride) * feature_stride + shifts_x, shifts_y = np.meshgrid(shifts_x, shifts_y) + + # Enumerate combinations of shifts, widths, and heights + box_widths, box_centers_x = np.meshgrid(widths, shifts_x) + box_heights, box_centers_y = np.meshgrid(heights, shifts_y) + + # Reshape to get a list of (y, x) and a list of (h, w) + box_centers = np.stack( + [box_centers_y, box_centers_x], axis=2).reshape([-1, 2]) + box_sizes = np.stack([box_heights, box_widths], axis=2).reshape([-1, 2]) + + # Convert to corner coordinates (y1, x1, y2, x2) + boxes = np.concatenate([box_centers - 0.5 * box_sizes, + box_centers + 0.5 * box_sizes], axis=1) + return boxes + + +def generate_pyramid_anchors(scales, ratios, feature_shapes, feature_strides, + anchor_stride): + """Generate anchors at different levels of a feature pyramid. Each scale + is associated with a level of the pyramid, but each ratio is used in + all levels of the pyramid. + + Returns: + anchors: [N, (y1, x1, y2, x2)]. All generated anchors in one array. Sorted + with the same order of the given scales. So, anchors of scale[0] come + first, then anchors of scale[1], and so on. + """ + # Anchors + # [anchor_count, (y1, x1, y2, x2)] + anchors = [] + for i in range(len(scales)): + anchors.append(generate_anchors(scales[i], ratios, feature_shapes[i], + feature_strides[i], anchor_stride)) + return np.concatenate(anchors, axis=0) + + +############################################################ +# Miscellaneous +############################################################ + +def trim_zeros(x): + """It's common to have tensors larger than the available data and + pad with zeros. This function removes rows that are all zeros. + + x: [rows, columns]. + """ + assert len(x.shape) == 2 + return x[~np.all(x == 0, axis=1)] + + +def compute_matches(gt_boxes, gt_class_ids, gt_masks, + pred_boxes, pred_class_ids, pred_scores, pred_masks, + iou_threshold=0.5, score_threshold=0.0): + """Finds matches between prediction and ground truth instances. + + Returns: + gt_match: 1-D array. For each GT box it has the index of the matched + predicted box. + pred_match: 1-D array. For each predicted box, it has the index of + the matched ground truth box. + overlaps: [pred_boxes, gt_boxes] IoU overlaps. + """ + # Trim zero padding + # TODO: cleaner to do zero unpadding upstream + gt_boxes = trim_zeros(gt_boxes) + gt_masks = gt_masks[..., :gt_boxes.shape[0]] + pred_boxes = trim_zeros(pred_boxes) + pred_scores = pred_scores[:pred_boxes.shape[0]] + # Sort predictions by score from high to low + indices = np.argsort(pred_scores)[::-1] + pred_boxes = pred_boxes[indices] + pred_class_ids = pred_class_ids[indices] + pred_scores = pred_scores[indices] + pred_masks = pred_masks[..., indices] + + # Compute IoU overlaps [pred_masks, gt_masks] + overlaps = compute_overlaps_masks(pred_masks, gt_masks) + + # Loop through predictions and find matching ground truth boxes + match_count = 0 + pred_match = -1 * np.ones([pred_boxes.shape[0]]) + gt_match = -1 * np.ones([gt_boxes.shape[0]]) + for i in range(len(pred_boxes)): + # Find best matching ground truth box + # 1. Sort matches by score + sorted_ixs = np.argsort(overlaps[i])[::-1] + # 2. Remove low scores + low_score_idx = np.where(overlaps[i, sorted_ixs] < score_threshold)[0] + if low_score_idx.size > 0: + sorted_ixs = sorted_ixs[:low_score_idx[0]] + # 3. Find the match + for j in sorted_ixs: + # If ground truth box is already matched, go to next one + if gt_match[j] > -1: + continue + # If we reach IoU smaller than the threshold, end the loop + iou = overlaps[i, j] + if iou < iou_threshold: + break + # Do we have a match? + if pred_class_ids[i] == gt_class_ids[j]: + match_count += 1 + gt_match[j] = i + pred_match[i] = j + break + + return gt_match, pred_match, overlaps + + +def compute_ap(gt_boxes, gt_class_ids, gt_masks, + pred_boxes, pred_class_ids, pred_scores, pred_masks, + iou_threshold=0.5): + """Compute Average Precision at a set IoU threshold (default 0.5). + + Returns: + mAP: Mean Average Precision + precisions: List of precisions at different class score thresholds. + recalls: List of recall values at different class score thresholds. + overlaps: [pred_boxes, gt_boxes] IoU overlaps. + """ + # Get matches and overlaps + gt_match, pred_match, overlaps = compute_matches( + gt_boxes, gt_class_ids, gt_masks, + pred_boxes, pred_class_ids, pred_scores, pred_masks, + iou_threshold) + + # Compute precision and recall at each prediction box step + precisions = np.cumsum(pred_match > -1) / (np.arange(len(pred_match)) + 1) + recalls = np.cumsum(pred_match > -1).astype(np.float32) / len(gt_match) + + # Pad with start and end values to simplify the math + precisions = np.concatenate([[0], precisions, [0]]) + recalls = np.concatenate([[0], recalls, [1]]) + + # Ensure precision values decrease but don't increase. This way, the + # precision value at each recall threshold is the maximum it can be + # for all following recall thresholds, as specified by the VOC paper. + for i in range(len(precisions) - 2, -1, -1): + precisions[i] = np.maximum(precisions[i], precisions[i + 1]) + + # Compute mean AP over recall range + indices = np.where(recalls[:-1] != recalls[1:])[0] + 1 + mAP = np.sum((recalls[indices] - recalls[indices - 1]) * + precisions[indices]) + + return mAP, precisions, recalls, overlaps + + +def compute_ap_range(gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + iou_thresholds=None, verbose=1): + """Compute AP over a range or IoU thresholds. Default range is 0.5-0.95.""" + # Default is 0.5 to 0.95 with increments of 0.05 + iou_thresholds = iou_thresholds or np.arange(0.5, 1.0, 0.05) + + # Compute AP over range of IoU thresholds + AP = [] + for iou_threshold in iou_thresholds: + ap, precisions, recalls, overlaps =\ + compute_ap(gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + iou_threshold=iou_threshold) + if verbose: + print("AP @{:.2f}:\t {:.3f}".format(iou_threshold, ap)) + AP.append(ap) + AP = np.array(AP).mean() + if verbose: + print("AP @{:.2f}-{:.2f}:\t {:.3f}".format( + iou_thresholds[0], iou_thresholds[-1], AP)) + return AP + + +def compute_recall(pred_boxes, gt_boxes, iou): + """Compute the recall at the given IoU threshold. It's an indication + of how many GT boxes were found by the given prediction boxes. + + pred_boxes: [N, (y1, x1, y2, x2)] in image coordinates + gt_boxes: [N, (y1, x1, y2, x2)] in image coordinates + """ + # Measure overlaps + overlaps = compute_overlaps(pred_boxes, gt_boxes) + iou_max = np.max(overlaps, axis=1) + iou_argmax = np.argmax(overlaps, axis=1) + positive_ids = np.where(iou_max >= iou)[0] + matched_gt_boxes = iou_argmax[positive_ids] + + recall = len(set(matched_gt_boxes)) / gt_boxes.shape[0] + return recall, positive_ids + + +# ## Batch Slicing +# Some custom layers support a batch size of 1 only, and require a lot of work +# to support batches greater than 1. This function slices an input tensor +# across the batch dimension and feeds batches of size 1. Effectively, +# an easy way to support batches > 1 quickly with little code modification. +# In the long run, it's more efficient to modify the code to support large +# batches and getting rid of this function. Consider this a temporary solution +def batch_slice(inputs, graph_fn, batch_size, names=None): + """Splits inputs into slices and feeds each slice to a copy of the given + computation graph and then combines the results. It allows you to run a + graph on a batch of inputs even if the graph is written to support one + instance only. + + inputs: list of tensors. All must have the same first dimension length + graph_fn: A function that returns a TF tensor that's part of a graph. + batch_size: number of slices to divide the data into. + names: If provided, assigns names to the resulting tensors. + """ + if not isinstance(inputs, list): + inputs = [inputs] + + outputs = [] + for i in range(batch_size): + inputs_slice = [x[i] for x in inputs] + output_slice = graph_fn(*inputs_slice) + if not isinstance(output_slice, (tuple, list)): + output_slice = [output_slice] + outputs.append(output_slice) + # Change outputs from a list of slices where each is + # a list of outputs to a list of outputs and each has + # a list of slices + outputs = list(zip(*outputs)) + + if names is None: + names = [None] * len(outputs) + + result = [tf.stack(o, axis=0, name=n) + for o, n in zip(outputs, names)] + if len(result) == 1: + result = result[0] + + return result + + +def download_trained_weights(coco_model_path, verbose=1): + """Download COCO trained weights from Releases. + + coco_model_path: local path of COCO trained weights + """ + if verbose > 0: + print("Downloading pretrained model to " + coco_model_path + " ...") + with urllib.request.urlopen(COCO_MODEL_URL) as resp, open(coco_model_path, 'wb') as out: + shutil.copyfileobj(resp, out) + if verbose > 0: + print("... done downloading pretrained model!") + + +def norm_boxes(boxes, shape): + """Converts boxes from pixel coordinates to normalized coordinates. + boxes: [N, (y1, x1, y2, x2)] in pixel coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [N, (y1, x1, y2, x2)] in normalized coordinates + """ + h, w = shape + scale = np.array([h - 1, w - 1, h - 1, w - 1]) + shift = np.array([0, 0, 1, 1]) + return np.divide((boxes - shift), scale).astype(np.float32) + + +def denorm_boxes(boxes, shape): + """Converts boxes from normalized coordinates to pixel coordinates. + boxes: [N, (y1, x1, y2, x2)] in normalized coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [N, (y1, x1, y2, x2)] in pixel coordinates + """ + h, w = shape + scale = np.array([h - 1, w - 1, h - 1, w - 1]) + shift = np.array([0, 0, 1, 1]) + return np.around(np.multiply(boxes, scale) + shift).astype(np.int32) + + +def resize(image, output_shape, order=1, mode='constant', cval=0, clip=True, + preserve_range=False, anti_aliasing=False, anti_aliasing_sigma=None): + """A wrapper for Scikit-Image resize(). + + Scikit-Image generates warnings on every call to resize() if it doesn't + receive the right parameters. The right parameters depend on the version + of skimage. This solves the problem by using different parameters per + version. And it provides a central place to control resizing defaults. + """ + if LooseVersion(skimage.__version__) >= LooseVersion("0.14"): + # New in 0.14: anti_aliasing. Default it to False for backward + # compatibility with skimage 0.13. + return skimage.transform.resize( + image, output_shape, + order=order, mode=mode, cval=cval, clip=clip, + preserve_range=preserve_range, anti_aliasing=anti_aliasing, + anti_aliasing_sigma=anti_aliasing_sigma) + else: + return skimage.transform.resize( + image, output_shape, + order=order, mode=mode, cval=cval, clip=clip, + preserve_range=preserve_range) diff --git a/mask_rcnn/mrcnn/visualize.py b/mask_rcnn/mrcnn/visualize.py new file mode 100644 index 00000000..80e5ef58 --- /dev/null +++ b/mask_rcnn/mrcnn/visualize.py @@ -0,0 +1,502 @@ +""" +Mask R-CNN +Display and Visualization Functions. + +Copyright (c) 2017 Matterport, Inc. +Licensed under the MIT License (see LICENSE for details) +Written by Waleed Abdulla +""" +import warnings +warnings.filterwarnings('ignore', category=DeprecationWarning) +warnings.filterwarnings('ignore', category=FutureWarning) +import os +import sys +import random +import itertools +import colorsys + +import numpy as np +from skimage.measure import find_contours +import matplotlib.pyplot as plt +from matplotlib import patches, lines +from matplotlib.patches import Polygon +import IPython.display + +# Root directory of the project +ROOT_DIR = os.path.abspath("../") + +# Import Mask RCNN +sys.path.append(ROOT_DIR) # To find local version of the library +from mrcnn import utils + + +############################################################ +# Visualization +############################################################ + +def display_images(images, titles=None, cols=4, cmap=None, norm=None, + interpolation=None): + """Display the given set of images, optionally with titles. + images: list or array of image tensors in HWC format. + titles: optional. A list of titles to display with each image. + cols: number of images per row + cmap: Optional. Color map to use. For example, "Blues". + norm: Optional. A Normalize instance to map values to colors. + interpolation: Optional. Image interpolation to use for display. + """ + titles = titles if titles is not None else [""] * len(images) + rows = len(images) // cols + 1 + plt.figure(figsize=(14, 14 * rows // cols)) + i = 1 + for image, title in zip(images, titles): + plt.subplot(rows, cols, i) + plt.title(title, fontsize=9) + plt.axis('off') + plt.imshow(image.astype(np.uint8), cmap=cmap, + norm=norm, interpolation=interpolation) + i += 1 + plt.show() + + +def random_colors(N, bright=True): + """ + Generate random colors. + To get visually distinct colors, generate them in HSV space then + convert to RGB. + """ + brightness = 1.0 if bright else 0.7 + hsv = [(i / N, 1, brightness) for i in range(N)] + colors = list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv)) + random.shuffle(colors) + return colors + + +def apply_mask(image, mask, color, alpha=0.5): + """Apply the given mask to the image. + """ + for c in range(3): + image[:, :, c] = np.where(mask == 1, + image[:, :, c] * + (1 - alpha) + alpha * color[c] * 255, + image[:, :, c]) + return image + + +def display_instances(image, boxes, masks, class_ids, class_names, + scores=None, title="", + figsize=(16, 16), ax=None, + show_mask=True, show_bbox=True, + colors=None, captions=None): + """ + boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates. + masks: [height, width, num_instances] + class_ids: [num_instances] + class_names: list of class names of the dataset + scores: (optional) confidence scores for each box + title: (optional) Figure title + show_mask, show_bbox: To show masks and bounding boxes or not + figsize: (optional) the size of the image + colors: (optional) An array or colors to use with each object + captions: (optional) A list of strings to use as captions for each object + """ + # Number of instances + N = boxes.shape[0] + if not N: + print("\n*** No instances to display *** \n") + else: + assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0] + + # If no axis is passed, create one and automatically call show() + auto_show = False + if not ax: + _, ax = plt.subplots(1, figsize=figsize) + auto_show = True + + # Generate random colors + colors = colors or random_colors(N) + + # Show area outside image boundaries. + height, width = image.shape[:2] + ax.set_ylim(height + 10, -10) + ax.set_xlim(-10, width + 10) + ax.axis('off') + ax.set_title(title) + + masked_image = image.astype(np.uint32).copy() + for i in range(N): + color = colors[i] + + # Bounding box + if not np.any(boxes[i]): + # Skip this instance. Has no bbox. Likely lost in image cropping. + continue + y1, x1, y2, x2 = boxes[i] + if show_bbox: + p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, + alpha=0.7, linestyle="dashed", + edgecolor=color, facecolor='none') + ax.add_patch(p) + + # Label + if not captions: + class_id = class_ids[i] + score = scores[i] if scores is not None else None + label = class_names[class_id] + caption = "{} {:.3f}".format(label, score) if score else label + else: + caption = captions[i] + ax.text(x1, y1 + 8, caption, + color='w', size=11, backgroundcolor="none") + + # Mask + mask = masks[:, :, i] + if show_mask: + masked_image = apply_mask(masked_image, mask, color) + + # Mask Polygon + # Pad to ensure proper polygons for masks that touch image edges. + padded_mask = np.zeros( + (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8) + padded_mask[1:-1, 1:-1] = mask + contours = find_contours(padded_mask, 0.5) + for verts in contours: + # Subtract the padding and flip (y, x) to (x, y) + verts = np.fliplr(verts) - 1 + p = Polygon(verts, facecolor="none", edgecolor=color) + ax.add_patch(p) + ax.imshow(masked_image.astype(np.uint8)) + if auto_show: + plt.show() + + +def display_differences(image, + gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + class_names, title="", ax=None, + show_mask=True, show_box=True, + iou_threshold=0.5, score_threshold=0.5): + """Display ground truth and prediction instances on the same image.""" + # Match predictions to ground truth + gt_match, pred_match, overlaps = utils.compute_matches( + gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + iou_threshold=iou_threshold, score_threshold=score_threshold) + # Ground truth = green. Predictions = red + colors = [(0, 1, 0, .8)] * len(gt_match)\ + + [(1, 0, 0, 1)] * len(pred_match) + # Concatenate GT and predictions + class_ids = np.concatenate([gt_class_id, pred_class_id]) + scores = np.concatenate([np.zeros([len(gt_match)]), pred_score]) + boxes = np.concatenate([gt_box, pred_box]) + masks = np.concatenate([gt_mask, pred_mask], axis=-1) + # Captions per instance show score/IoU + captions = ["" for m in gt_match] + ["{:.2f} / {:.2f}".format( + pred_score[i], + (overlaps[i, int(pred_match[i])] + if pred_match[i] > -1 else overlaps[i].max())) + for i in range(len(pred_match))] + # Set title if not provided + title = title or "Ground Truth and Detections\n GT=green, pred=red, captions: score/IoU" + # Display + display_instances( + image, + boxes, masks, class_ids, + class_names, scores, ax=ax, + show_bbox=show_box, show_mask=show_mask, + colors=colors, captions=captions, + title=title) + + +def draw_rois(image, rois, refined_rois, mask, class_ids, class_names, limit=10): + """ + anchors: [n, (y1, x1, y2, x2)] list of anchors in image coordinates. + proposals: [n, 4] the same anchors but refined to fit objects better. + """ + masked_image = image.copy() + + # Pick random anchors in case there are too many. + ids = np.arange(rois.shape[0], dtype=np.int32) + ids = np.random.choice( + ids, limit, replace=False) if ids.shape[0] > limit else ids + + fig, ax = plt.subplots(1, figsize=(12, 12)) + if rois.shape[0] > limit: + plt.title("Showing {} random ROIs out of {}".format( + len(ids), rois.shape[0])) + else: + plt.title("{} ROIs".format(len(ids))) + + # Show area outside image boundaries. + ax.set_ylim(image.shape[0] + 20, -20) + ax.set_xlim(-50, image.shape[1] + 20) + ax.axis('off') + + for i, id in enumerate(ids): + color = np.random.rand(3) + class_id = class_ids[id] + # ROI + y1, x1, y2, x2 = rois[id] + p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, + edgecolor=color if class_id else "gray", + facecolor='none', linestyle="dashed") + ax.add_patch(p) + # Refined ROI + if class_id: + ry1, rx1, ry2, rx2 = refined_rois[id] + p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2, + edgecolor=color, facecolor='none') + ax.add_patch(p) + # Connect the top-left corners of the anchor and proposal for easy visualization + ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color)) + + # Label + label = class_names[class_id] + ax.text(rx1, ry1 + 8, "{}".format(label), + color='w', size=11, backgroundcolor="none") + + # Mask + m = utils.unmold_mask(mask[id], rois[id] + [:4].astype(np.int32), image.shape) + masked_image = apply_mask(masked_image, m, color) + + ax.imshow(masked_image) + + # Print stats + print("Positive ROIs: ", class_ids[class_ids > 0].shape[0]) + print("Negative ROIs: ", class_ids[class_ids == 0].shape[0]) + print("Positive Ratio: {:.2f}".format( + class_ids[class_ids > 0].shape[0] / class_ids.shape[0])) + + +# TODO: Replace with matplotlib equivalent? +def draw_box(image, box, color): + """Draw 3-pixel width bounding boxes on the given image array. + color: list of 3 int values for RGB. + """ + y1, x1, y2, x2 = box + image[y1:y1 + 2, x1:x2] = color + image[y2:y2 + 2, x1:x2] = color + image[y1:y2, x1:x1 + 2] = color + image[y1:y2, x2:x2 + 2] = color + return image + + +def display_top_masks(image, mask, class_ids, class_names, limit=4): + """Display the given image and the top few class masks.""" + to_display = [] + titles = [] + to_display.append(image) + titles.append("H x W={}x{}".format(image.shape[0], image.shape[1])) + # Pick top prominent classes in this image + unique_class_ids = np.unique(class_ids) + mask_area = [np.sum(mask[:, :, np.where(class_ids == i)[0]]) + for i in unique_class_ids] + top_ids = [v[0] for v in sorted(zip(unique_class_ids, mask_area), + key=lambda r: r[1], reverse=True) if v[1] > 0] + # Generate images and titles + for i in range(limit): + class_id = top_ids[i] if i < len(top_ids) else -1 + # Pull masks of instances belonging to the same class. + m = mask[:, :, np.where(class_ids == class_id)[0]] + m = np.sum(m * np.arange(1, m.shape[-1] + 1), -1) + to_display.append(m) + titles.append(class_names[class_id] if class_id != -1 else "-") + display_images(to_display, titles=titles, cols=limit + 1, cmap="Blues_r") + + +def plot_precision_recall(AP, precisions, recalls): + """Draw the precision-recall curve. + + AP: Average precision at IoU >= 0.5 + precisions: list of precision values + recalls: list of recall values + """ + # Plot the Precision-Recall curve + _, ax = plt.subplots(1) + ax.set_title("Precision-Recall Curve. AP@50 = {:.3f}".format(AP)) + ax.set_ylim(0, 1.1) + ax.set_xlim(0, 1.1) + _ = ax.plot(recalls, precisions) + + +def plot_overlaps(gt_class_ids, pred_class_ids, pred_scores, + overlaps, class_names, threshold=0.5): + """Draw a grid showing how ground truth objects are classified. + gt_class_ids: [N] int. Ground truth class IDs + pred_class_id: [N] int. Predicted class IDs + pred_scores: [N] float. The probability scores of predicted classes + overlaps: [pred_boxes, gt_boxes] IoU overlaps of predictions and GT boxes. + class_names: list of all class names in the dataset + threshold: Float. The prediction probability required to predict a class + """ + gt_class_ids = gt_class_ids[gt_class_ids != 0] + pred_class_ids = pred_class_ids[pred_class_ids != 0] + + plt.figure(figsize=(12, 10)) + plt.imshow(overlaps, interpolation='nearest', cmap=plt.cm.Blues) + plt.yticks(np.arange(len(pred_class_ids)), + ["{} ({:.2f})".format(class_names[int(id)], pred_scores[i]) + for i, id in enumerate(pred_class_ids)]) + plt.xticks(np.arange(len(gt_class_ids)), + [class_names[int(id)] for id in gt_class_ids], rotation=90) + + thresh = overlaps.max() / 2. + for i, j in itertools.product(range(overlaps.shape[0]), + range(overlaps.shape[1])): + text = "" + if overlaps[i, j] > threshold: + text = "match" if gt_class_ids[j] == pred_class_ids[i] else "wrong" + color = ("white" if overlaps[i, j] > thresh + else "black" if overlaps[i, j] > 0 + else "grey") + plt.text(j, i, "{:.3f}\n{}".format(overlaps[i, j], text), + horizontalalignment="center", verticalalignment="center", + fontsize=9, color=color) + + plt.tight_layout() + plt.xlabel("Ground Truth") + plt.ylabel("Predictions") + + +def draw_boxes(image, boxes=None, refined_boxes=None, + masks=None, captions=None, visibilities=None, + title="", ax=None): + """Draw bounding boxes and segmentation masks with different + customizations. + + boxes: [N, (y1, x1, y2, x2, class_id)] in image coordinates. + refined_boxes: Like boxes, but draw with solid lines to show + that they're the result of refining 'boxes'. + masks: [N, height, width] + captions: List of N titles to display on each box + visibilities: (optional) List of values of 0, 1, or 2. Determine how + prominent each bounding box should be. + title: An optional title to show over the image + ax: (optional) Matplotlib axis to draw on. + """ + # Number of boxes + assert boxes is not None or refined_boxes is not None + N = boxes.shape[0] if boxes is not None else refined_boxes.shape[0] + + # Matplotlib Axis + if not ax: + _, ax = plt.subplots(1, figsize=(12, 12)) + + # Generate random colors + colors = random_colors(N) + + # Show area outside image boundaries. + margin = image.shape[0] // 10 + ax.set_ylim(image.shape[0] + margin, -margin) + ax.set_xlim(-margin, image.shape[1] + margin) + ax.axis('off') + + ax.set_title(title) + + masked_image = image.astype(np.uint32).copy() + for i in range(N): + # Box visibility + visibility = visibilities[i] if visibilities is not None else 1 + if visibility == 0: + color = "gray" + style = "dotted" + alpha = 0.5 + elif visibility == 1: + color = colors[i] + style = "dotted" + alpha = 1 + elif visibility == 2: + color = colors[i] + style = "solid" + alpha = 1 + + # Boxes + if boxes is not None: + if not np.any(boxes[i]): + # Skip this instance. Has no bbox. Likely lost in cropping. + continue + y1, x1, y2, x2 = boxes[i] + p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, + alpha=alpha, linestyle=style, + edgecolor=color, facecolor='none') + ax.add_patch(p) + + # Refined boxes + if refined_boxes is not None and visibility > 0: + ry1, rx1, ry2, rx2 = refined_boxes[i].astype(np.int32) + p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2, + edgecolor=color, facecolor='none') + ax.add_patch(p) + # Connect the top-left corners of the anchor and proposal + if boxes is not None: + ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color)) + + # Captions + if captions is not None: + caption = captions[i] + # If there are refined boxes, display captions on them + if refined_boxes is not None: + y1, x1, y2, x2 = ry1, rx1, ry2, rx2 + ax.text(x1, y1, caption, size=11, verticalalignment='top', + color='w', backgroundcolor="none", + bbox={'facecolor': color, 'alpha': 0.5, + 'pad': 2, 'edgecolor': 'none'}) + + # Masks + if masks is not None: + mask = masks[:, :, i] + masked_image = apply_mask(masked_image, mask, color) + # Mask Polygon + # Pad to ensure proper polygons for masks that touch image edges. + padded_mask = np.zeros( + (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8) + padded_mask[1:-1, 1:-1] = mask + contours = find_contours(padded_mask, 0.5) + for verts in contours: + # Subtract the padding and flip (y, x) to (x, y) + verts = np.fliplr(verts) - 1 + p = Polygon(verts, facecolor="none", edgecolor=color) + ax.add_patch(p) + ax.imshow(masked_image.astype(np.uint8)) + + +def display_table(table): + """Display values in a table format. + table: an iterable of rows, and each row is an iterable of values. + """ + html = "" + for row in table: + row_html = "" + for col in row: + row_html += "{:40}".format(str(col)) + html += "" + row_html + "" + html = "" + html + "
    " + IPython.display.display(IPython.display.HTML(html)) + + +def display_weight_stats(model): + """Scans all the weights in the model and returns a list of tuples + that contain stats about each weight. + """ + layers = model.get_trainable_layers() + table = [["WEIGHT NAME", "SHAPE", "MIN", "MAX", "STD"]] + for l in layers: + weight_values = l.get_weights() # list of Numpy arrays + weight_tensors = l.weights # list of TF tensors + for i, w in enumerate(weight_values): + weight_name = weight_tensors[i].name + # Detect problematic layers. Exclude biases of conv layers. + alert = "" + if w.min() == w.max() and not (l.__class__.__name__ == "Conv2D" and i == 1): + alert += "*** dead?" + if np.abs(w.min()) > 1000 or np.abs(w.max()) > 1000: + alert += "*** Overflow?" + # Add row + table.append([ + weight_name + alert, + str(w.shape), + "{:+9.4f}".format(w.min()), + "{:+10.4f}".format(w.max()), + "{:+9.4f}".format(w.std()), + ]) + display_table(table) diff --git a/mask_rcnn/requirements.txt b/mask_rcnn/requirements.txt new file mode 100644 index 00000000..22e883b7 --- /dev/null +++ b/mask_rcnn/requirements.txt @@ -0,0 +1,12 @@ +numpy +scipy +Pillow +cython +matplotlib +scikit-image +tensorflow-gpu +keras==2.2.5 +opencv-python +h5py +imgaug +IPython[all] \ No newline at end of file diff --git a/mask_rcnn/score.py b/mask_rcnn/score.py new file mode 100644 index 00000000..ad9bc731 --- /dev/null +++ b/mask_rcnn/score.py @@ -0,0 +1,300 @@ +from mrcnn.model import log +import mrcnn.model as modellib +from mrcnn.visualize import display_images +import mrcnn.visualize as visualize +import mrcnn.utils as utils +from mrcnn.config import Config +import sys +import math +import re +import time +import numpy as np +import tensorflow as tf +import matplotlib +import matplotlib.pyplot as plt +import matplotlib.patches as patches +import os +import sys +import json +import datetime +import skimage.draw +import cv2 +import argparse + +# Device to load the neural network on. +# Useful if you're training a model on the same +# machine, in which case use CPU and leave the +# GPU for training. +DEVICE = "/cpu:0" # /cpu:0 or /gpu:0 + +ACCEPTED_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif'] +CLASS_NAMES = ['BG', 'outerbox', 'innerbox', 'item_sq', 'item_rect', 'item_rect_slim', 'item_circ'] +ITEM_NAMES = CLASS_NAMES[3:] + +class PPConfig(Config): + """Configuration for training on the toy dataset. + Derives from the base Config class and overrides some values. + """ + # Give the configuration a recognizable name + NAME = "pointless_package" + + # We use a GPU with 12GB memory, which can fit two images. + # Adjust down if you use a smaller GPU. + IMAGES_PER_GPU = 1 + + # Number of classes (including background) + # Background + outerbox + innerbox + item_rect + item_rect_slim + item_sq + item_circ + NUM_CLASSES = 1 + 6 + + # Number of training steps per epoch + STEPS_PER_EPOCH = 100 + + # Skip detections with < 90% confidence + DETECTION_MIN_CONFIDENCE = 0.75 + +class PPDataset(utils.Dataset): + + def load_dataset(self, dataset_dir, subset): + """Load a subset of the Balloon dataset. + dataset_dir: Root directory of the dataset. + subset: Subset to load: train or val + """ + # Add classes. We have only one class to add. + self.add_class("pointless_package", 1, "outerbox") + self.add_class("pointless_package", 2, "innerbox") + self.add_class("pointless_package", 3, "item_sq") + self.add_class("pointless_package", 4, "item_rect") + self.add_class("pointless_package", 5, "item_rect_slim") + self.add_class("pointless_package", 6, "item_circ") + + # Train or validation dataset? + assert subset in ["train", "val"] + dataset_dir = os.path.join(dataset_dir, subset) + + # Load annotations + # VGG Image Annotator (up to version 1.6) saves each image in the form: + # { 'filename': '28503151_5b5b7ec140_b.jpg', + # 'regions': { + # '0': { + # 'region_attributes': {}, + # 'shape_attributes': { + # 'all_points_x': [...], + # 'all_points_y': [...], + # 'name': 'polygon'}}, + # ... more regions ... + # }, + # 'size': 100202 + # } + # We mostly care about the x and y coordinates of each region + # Note: In VIA 2.0, regions was changed from a dict to a list. + annotations = json.load( + open(os.path.join(dataset_dir, "via_region_data.json"))) + annotations = list(annotations.values()) # don't need the dict keys + + # The VIA tool saves images in the JSON even if they don't have any + # annotations. Skip unannotated images. + annotations = [a for a in annotations if a['regions']] + + # Add images + for a in annotations: + # Get the x, y coordinaets of points of the polygons that make up + # the outline of each object instance. These are stores in the + # shape_attributes (see json format above) + # The if condition is needed to support VIA versions 1.x and 2.x. + if type(a['regions']) is dict: + polygons = [r['shape_attributes'] + for r in a['regions'].values()] + else: + polygons = [r['shape_attributes'] for r in a['regions']] + + # load_mask() needs the image size to convert polygons to masks. + # Unfortunately, VIA doesn't include it in JSON, so we must read + # the image. This is only managable since the dataset is tiny. + image_path = os.path.join(dataset_dir, a['filename']) + image = skimage.io.imread(image_path) + height, width = image.shape[:2] + + class_list = [r['region_attributes'] for r in a['regions']] + + self.add_image( + "pointless_package", + image_id=a['filename'], # use file name as a unique image id + path=image_path, + width=width, height=height, + class_list=class_list, + polygons=polygons) + + def load_mask(self, image_id): + """Generate instance masks for an image. + Returns: + masks: A bool array of shape [height, width, instance count] with + one mask per instance. + class_ids: a 1D array of class IDs of the instance masks. + """ + class_ids = list() + # If not a pointless_package dataset image, delegate to parent class. + image_info = self.image_info[image_id] + # if image_info["source"] != "pointless_package": + # return super(self.__class__, self).load_mask(image_id) + + # Convert polygons to a bitmap mask of shape + # [height, width, instance_count] + info = self.image_info[image_id] + # print("\n\n\nIMAGE INFO:", info, "\n\n\n\n") + + for box_type in info['class_list']: + # print(box_type['name']) + class_ids.append(self.class_names.index(str(box_type['name']))) + # print(class_ids) + # print(self.class_names) + + mask = np.zeros([info["height"], info["width"], len(info["polygons"])], + dtype=np.uint8) + for i, p in enumerate(info["polygons"]): + # Get indexes of pixels inside the polygon and set them to 1 + rr, cc = skimage.draw.polygon(p['all_points_y'], p['all_points_x']) + mask[rr, cc, i] = 1 + # Return mask, and array of class IDs of each instance. Since we have + # one class ID only, we return an array of 1s + return mask.astype(np.bool), np.asarray(class_ids, dtype=np.int32) + + def image_reference(self, image_id): + """Return the path of the image.""" + info = self.image_info[image_id] + if info["source"] == "pointless_package": + return info["path"] + else: + super(self.__class__, self).image_reference(image_id) + +config = PPConfig() +ROOT_DIR = os.getcwd() +PP_DIR = os.path.join(ROOT_DIR, "./") + +# Override the training configurations with a few +# changes for inferencing. +class InferenceConfig(config.__class__): + # Run detection on one image at a time + GPU_COUNT = 1 + IMAGES_PER_GPU = 1 + +config = InferenceConfig() + +def get_ax(rows=1, cols=1, size=16): + """Return a Matplotlib Axes array to be used in + all visualizations in the notebook. Provide a + central point to control graph sizes. + + Adjust the size attribute to control how big to render images + """ + _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows)) + return ax + +description = "Simple script that takes a trained MASK R-CNN model (.h5), a pointless packaging image and then runs generates score of the packaging purely based on the area; using the provided model." + +def parse_args(): + parser = argparse.ArgumentParser(description=description) + parser.add_argument('-m', '--model', required=True, + help='Absolute/Relative path to the MASK R-CNN Model', dest='model_src') + g = parser.add_mutually_exclusive_group() + g.add_argument('-i', '--img', required=False, + help='Absolute/Relative path of the image. Cannot include --dir argument.', dest='img_src') + g.add_argument('-d', '--dir', required=False, + help='Absolute/Relative path of the directory containing the images. Cannot include --img argument.', dest='dir_src') + parser.add_argument('-v', '--visualize', action='store_true', + help='Visualize the image.', dest='vis_arg') + + return parser.parse_args() + +def main(): + args = parse_args() + file_only = False if args.img_src is None else True + + if os.path.isfile(args.model_src) == False: + print("Model file does not exist. Please provide a model.") + raise SystemExit + + if file_only: + if os.path.isfile(args.img_src) == False: + print("Please enter a valid image.") + raise SystemExit + img_src = os.path.splitext(args.img_src) # path of the image + else: + if os.path.isdir(args.dir_src) == False: + print("Please enter a valid directory.") + raise SystemExit + + model_src = os.path.splitext(args.model_src) # model path + + if model_src[1] != '.h5': + print("Not a valid model. Please enter .h5 models only.") + raise SystemExit + + ### SETUP MODEL ### + width = 300 + + # # Create model in inference mode + with tf.device(DEVICE): + model = modellib.MaskRCNN(mode="inference", model_dir='./models/', + config=config) + + # Load weights + print("Loading weights ", args.model_src) + model.load_weights(args.model_src, by_name=True) + + ### RUN INFERENCE ### + if file_only: + files = [args.img_src] + else: + try: + """ Get all files in the copied directory """ + files = os.listdir(args.dir_src) + except NotADirectoryError: + print("Invalid directory:", args.dir_src) + raise + + print("\n\n----------------------") + print(" PACKAGING SCORES ") + print("----------------------") + + for file in files: + if os.path.splitext(file)[1] not in ACCEPTED_EXTENSIONS: + # print(file + "is not a valid image.") + continue + image = cv2.imread(args.dir_src+file) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + + orig_dim = image.shape + new_dim = (int((width/orig_dim[0]) * orig_dim[1]), width) + image = cv2.resize( + image, new_dim, interpolation=cv2.INTER_AREA) + if orig_dim[0] > 300: + print("Image resized from", orig_dim[:2], " ->", image.shape[:2]) + + # Run object detection + results = model.detect([image], verbose=0) + + # Display results + r = results[0] + + N = r['rois'].shape[0] + class_ids = r['class_ids'] + masks = r['masks'] + + class_names = np.asarray(CLASS_NAMES) + + # area_occupation = [masks[:, :, i].sum() for i in range(N)] + area_occupation = masks.sum(axis=0).sum(axis=0) + map_items_to_area = list(map( + lambda x, y: x+': '+str(y), class_names[class_ids], area_occupation)) + print("{}: (ITEM: AREA) ->".format(os.path.basename(file)), map_items_to_area, "{}".format( + "" if set(ITEM_NAMES) & set(class_names[class_ids]) else "-> NO ITEMS FOUND!")) + + if args.vis_arg: + visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], + CLASS_NAMES, r['scores'], + title="Predictions") + + print("\n\nDone.\n") + +if __name__ == "__main__": + main() diff --git a/mask_rcnn/test_images/IMG_0.jpg b/mask_rcnn/test_images/IMG_0.jpg new file mode 100644 index 00000000..57b9d110 Binary files /dev/null and b/mask_rcnn/test_images/IMG_0.jpg differ diff --git a/mask_rcnn/test_images/IMG_1.jpg b/mask_rcnn/test_images/IMG_1.jpg new file mode 100644 index 00000000..eac21b38 Binary files /dev/null and b/mask_rcnn/test_images/IMG_1.jpg differ diff --git a/mask_rcnn/test_images/IMG_2.jpg b/mask_rcnn/test_images/IMG_2.jpg new file mode 100644 index 00000000..a67f55cb Binary files /dev/null and b/mask_rcnn/test_images/IMG_2.jpg differ diff --git a/mask_rcnn/test_images/IMG_26.jpg b/mask_rcnn/test_images/IMG_26.jpg new file mode 100644 index 00000000..b927984e Binary files /dev/null and b/mask_rcnn/test_images/IMG_26.jpg differ diff --git a/mask_rcnn/test_images/IMG_3.jpg b/mask_rcnn/test_images/IMG_3.jpg new file mode 100644 index 00000000..0258c891 Binary files /dev/null and b/mask_rcnn/test_images/IMG_3.jpg differ diff --git a/mask_rcnn/test_images/IMG_4.jpg b/mask_rcnn/test_images/IMG_4.jpg new file mode 100644 index 00000000..089831a3 Binary files /dev/null and b/mask_rcnn/test_images/IMG_4.jpg differ diff --git a/mask_rcnn/test_images/IMG_7.jpg b/mask_rcnn/test_images/IMG_7.jpg new file mode 100644 index 00000000..a19209ef Binary files /dev/null and b/mask_rcnn/test_images/IMG_7.jpg differ diff --git a/mask_rcnn/test_images/IMG_8.jpg b/mask_rcnn/test_images/IMG_8.jpg new file mode 100644 index 00000000..d846986b Binary files /dev/null and b/mask_rcnn/test_images/IMG_8.jpg differ diff --git a/mask_rcnn/test_images/IMG_9.jpg b/mask_rcnn/test_images/IMG_9.jpg new file mode 100644 index 00000000..0df3d43f Binary files /dev/null and b/mask_rcnn/test_images/IMG_9.jpg differ diff --git a/mask_rcnn/tf_servable/README.md b/mask_rcnn/tf_servable/README.md new file mode 100644 index 00000000..9d87f2af --- /dev/null +++ b/mask_rcnn/tf_servable/README.md @@ -0,0 +1,50 @@ +# CREDIT FOR THIS LIBRARY: https://github.com/bendangnuksung/mrcnn_serving_ready + +### MRCNN Model conversion +Script to convert [MatterPort Mask_RCNN](https://github.com/matterport/Mask_RCNN) Keras model to Tensorflow Frozen Graph and Tensorflow Serving Model. +Plus inferencing with GRPC or RESTAPI using Tensorflow Model Server. + + +### How to Run +1. Modify the path variables in 'user_config.py' +2. Run main.py + ```bash + python3 main.py + ``` + +#### For Custom Config class +If you have a different config class you can replace the existing config in 'main.py' +```python +# main.py +# Current config load +config = get_config() + +# replace it with your config class +config = your_custom_config_class + +``` + +### Inferencing +Follow once you finish converting it to a `saved_model` using the above code + +#### Tensorflow Model Server with GRPC and RESTAPI + +1. First run your `saved_model.pb` in Tensorflow Model Server, using: + ```bash + tensorflow_model_server --port=8500 --rest_api_port=8501 --model_name=mask --model_base_path=/path/to/saved_model/ + ``` +2. Modify the variables and add your Config Class if needed in `inferencing/saved_model_config.py`. No need to change if the saved_model is the default COCO model. +3. Then run the `inferencing/saved_model_inference.py` with the image path: + ```bash + # Set Python Path + export PYTHONPATH=$PYTHONPATH:$pwd + + # Run Inference with GRPC + python3 inferencing/saved_model_inference.py -t grpc -p test_image/monalisa.jpg + + # Run Inference with RESTAPI + python3 inferencing/saved_model_inference.py -t restapi -p test_image/monalisa.jpg + ``` + +### Acknowledgement +Thanks to [@rahulgullan](https://github.com/rahulgullan) for RESTAPI client code. \ No newline at end of file diff --git a/mask_rcnn/tf_servable/coco.py b/mask_rcnn/tf_servable/coco.py new file mode 100644 index 00000000..5d172b5c --- /dev/null +++ b/mask_rcnn/tf_servable/coco.py @@ -0,0 +1,534 @@ +""" +Mask R-CNN +Configurations and data loading code for MS COCO. + +Copyright (c) 2017 Matterport, Inc. +Licensed under the MIT License (see LICENSE for details) +Written by Waleed Abdulla + +------------------------------------------------------------ + +Usage: import the module (see Jupyter notebooks for examples), or run from + the command line as such: + + # Train a new model starting from pre-trained COCO weights + python3 coco.py train --dataset=/path/to/coco/ --model=coco + + # Train a new model starting from ImageNet weights. Also auto download COCO dataset + python3 coco.py train --dataset=/path/to/coco/ --model=imagenet --download=True + + # Continue training a model that you had trained earlier + python3 coco.py train --dataset=/path/to/coco/ --model=/path/to/weights.h5 + + # Continue training the last model you trained + python3 coco.py train --dataset=/path/to/coco/ --model=last + + # Run COCO evaluatoin on the last model you trained + python3 coco.py evaluate --dataset=/path/to/coco/ --model=last +""" + +import os +import sys +import time +import numpy as np +import imgaug # https://github.com/aleju/imgaug (pip3 install imgaug) + +# Download and install the Python COCO tools from https://github.com/waleedka/coco +# That's a fork from the original https://github.com/pdollar/coco with a bug +# fix for Python 3. +# I submitted a pull request https://github.com/cocodataset/cocoapi/pull/50 +# If the PR is merged then use the original repo. +# Note: Edit PythonAPI/Makefile and replace "python" with "python3". +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +from pycocotools import mask as maskUtils + +import zipfile +import urllib.request +import shutil + +# Root directory of the project +ROOT_DIR = os.path.abspath("../../") + +# Import Mask RCNN +sys.path.append(ROOT_DIR) # To find local version of the library +from mrcnn.config import Config +from mrcnn import model as modellib, utils + +# Path to trained weights file +COCO_MODEL_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5") + +# Directory to save logs and model checkpoints, if not provided +# through the command line argument --logs +DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs") +DEFAULT_DATASET_YEAR = "2014" + +############################################################ +# Configurations +############################################################ + + +class CocoConfig(Config): + """Configuration for training on MS COCO. + Derives from the base Config class and overrides values specific + to the COCO dataset. + """ + # Give the configuration a recognizable name + NAME = "coco" + + # We use a GPU with 12GB memory, which can fit two images. + # Adjust down if you use a smaller GPU. + IMAGES_PER_GPU = 2 + + # Uncomment to train on 8 GPUs (default is 1) + # GPU_COUNT = 8 + + # Number of classes (including background) + NUM_CLASSES = 1 + 80 # COCO has 80 classes + + +############################################################ +# Dataset +############################################################ + +class CocoDataset(utils.Dataset): + def load_coco(self, dataset_dir, subset, year=DEFAULT_DATASET_YEAR, class_ids=None, + class_map=None, return_coco=False, auto_download=False): + """Load a subset of the COCO dataset. + dataset_dir: The root directory of the COCO dataset. + subset: What to load (train, val, minival, valminusminival) + year: What dataset year to load (2014, 2017) as a string, not an integer + class_ids: If provided, only loads images that have the given classes. + class_map: TODO: Not implemented yet. Supports maping classes from + different datasets to the same class ID. + return_coco: If True, returns the COCO object. + auto_download: Automatically download and unzip MS-COCO images and annotations + """ + + if auto_download is True: + self.auto_download(dataset_dir, subset, year) + + coco = COCO("{}/annotations/instances_{}{}.json".format(dataset_dir, subset, year)) + if subset == "minival" or subset == "valminusminival": + subset = "val" + image_dir = "{}/{}{}".format(dataset_dir, subset, year) + + # Load all classes or a subset? + if not class_ids: + # All classes + class_ids = sorted(coco.getCatIds()) + + # All images or a subset? + if class_ids: + image_ids = [] + for id in class_ids: + image_ids.extend(list(coco.getImgIds(catIds=[id]))) + # Remove duplicates + image_ids = list(set(image_ids)) + else: + # All images + image_ids = list(coco.imgs.keys()) + + # Add classes + for i in class_ids: + self.add_class("coco", i, coco.loadCats(i)[0]["name"]) + + # Add images + for i in image_ids: + self.add_image( + "coco", image_id=i, + path=os.path.join(image_dir, coco.imgs[i]['file_name']), + width=coco.imgs[i]["width"], + height=coco.imgs[i]["height"], + annotations=coco.loadAnns(coco.getAnnIds( + imgIds=[i], catIds=class_ids, iscrowd=None))) + if return_coco: + return coco + + def auto_download(self, dataDir, dataType, dataYear): + """Download the COCO dataset/annotations if requested. + dataDir: The root directory of the COCO dataset. + dataType: What to load (train, val, minival, valminusminival) + dataYear: What dataset year to load (2014, 2017) as a string, not an integer + Note: + For 2014, use "train", "val", "minival", or "valminusminival" + For 2017, only "train" and "val" annotations are available + """ + + # Setup paths and file names + if dataType == "minival" or dataType == "valminusminival": + imgDir = "{}/{}{}".format(dataDir, "val", dataYear) + imgZipFile = "{}/{}{}.zip".format(dataDir, "val", dataYear) + imgURL = "http://images.cocodataset.org/zips/{}{}.zip".format("val", dataYear) + else: + imgDir = "{}/{}{}".format(dataDir, dataType, dataYear) + imgZipFile = "{}/{}{}.zip".format(dataDir, dataType, dataYear) + imgURL = "http://images.cocodataset.org/zips/{}{}.zip".format(dataType, dataYear) + # print("Image paths:"); print(imgDir); print(imgZipFile); print(imgURL) + + # Create main folder if it doesn't exist yet + if not os.path.exists(dataDir): + os.makedirs(dataDir) + + # Download images if not available locally + if not os.path.exists(imgDir): + os.makedirs(imgDir) + print("Downloading images to " + imgZipFile + " ...") + with urllib.request.urlopen(imgURL) as resp, open(imgZipFile, 'wb') as out: + shutil.copyfileobj(resp, out) + print("... done downloading.") + print("Unzipping " + imgZipFile) + with zipfile.ZipFile(imgZipFile, "r") as zip_ref: + zip_ref.extractall(dataDir) + print("... done unzipping") + print("Will use images in " + imgDir) + + # Setup annotations data paths + annDir = "{}/annotations".format(dataDir) + if dataType == "minival": + annZipFile = "{}/instances_minival2014.json.zip".format(dataDir) + annFile = "{}/instances_minival2014.json".format(annDir) + annURL = "https://dl.dropboxusercontent.com/s/o43o90bna78omob/instances_minival2014.json.zip?dl=0" + unZipDir = annDir + elif dataType == "valminusminival": + annZipFile = "{}/instances_valminusminival2014.json.zip".format(dataDir) + annFile = "{}/instances_valminusminival2014.json".format(annDir) + annURL = "https://dl.dropboxusercontent.com/s/s3tw5zcg7395368/instances_valminusminival2014.json.zip?dl=0" + unZipDir = annDir + else: + annZipFile = "{}/annotations_trainval{}.zip".format(dataDir, dataYear) + annFile = "{}/instances_{}{}.json".format(annDir, dataType, dataYear) + annURL = "http://images.cocodataset.org/annotations/annotations_trainval{}.zip".format(dataYear) + unZipDir = dataDir + # print("Annotations paths:"); print(annDir); print(annFile); print(annZipFile); print(annURL) + + # Download annotations if not available locally + if not os.path.exists(annDir): + os.makedirs(annDir) + if not os.path.exists(annFile): + if not os.path.exists(annZipFile): + print("Downloading zipped annotations to " + annZipFile + " ...") + with urllib.request.urlopen(annURL) as resp, open(annZipFile, 'wb') as out: + shutil.copyfileobj(resp, out) + print("... done downloading.") + print("Unzipping " + annZipFile) + with zipfile.ZipFile(annZipFile, "r") as zip_ref: + zip_ref.extractall(unZipDir) + print("... done unzipping") + print("Will use annotations in " + annFile) + + def load_mask(self, image_id): + """Load instance masks for the given image. + + Different datasets use different ways to store masks. This + function converts the different mask format to one format + in the form of a bitmap [height, width, instances]. + + Returns: + masks: A bool array of shape [height, width, instance count] with + one mask per instance. + class_ids: a 1D array of class IDs of the instance masks. + """ + # If not a COCO image, delegate to parent class. + image_info = self.image_info[image_id] + if image_info["source"] != "coco": + return super(CocoDataset, self).load_mask(image_id) + + instance_masks = [] + class_ids = [] + annotations = self.image_info[image_id]["annotations"] + # Build mask of shape [height, width, instance_count] and list + # of class IDs that correspond to each channel of the mask. + for annotation in annotations: + class_id = self.map_source_class_id( + "coco.{}".format(annotation['category_id'])) + if class_id: + m = self.annToMask(annotation, image_info["height"], + image_info["width"]) + # Some objects are so small that they're less than 1 pixel area + # and end up rounded out. Skip those objects. + if m.max() < 1: + continue + # Is it a crowd? If so, use a negative class ID. + if annotation['iscrowd']: + # Use negative class ID for crowds + class_id *= -1 + # For crowd masks, annToMask() sometimes returns a mask + # smaller than the given dimensions. If so, resize it. + if m.shape[0] != image_info["height"] or m.shape[1] != image_info["width"]: + m = np.ones([image_info["height"], image_info["width"]], dtype=bool) + instance_masks.append(m) + class_ids.append(class_id) + + # Pack instance masks into an array + if class_ids: + mask = np.stack(instance_masks, axis=2).astype(np.bool) + class_ids = np.array(class_ids, dtype=np.int32) + return mask, class_ids + else: + # Call super class to return an empty mask + return super(CocoDataset, self).load_mask(image_id) + + def image_reference(self, image_id): + """Return a link to the image in the COCO Website.""" + info = self.image_info[image_id] + if info["source"] == "coco": + return "http://cocodataset.org/#explore?id={}".format(info["id"]) + else: + super(CocoDataset, self).image_reference(image_id) + + # The following two functions are from pycocotools with a few changes. + + def annToRLE(self, ann, height, width): + """ + Convert annotation which can be polygons, uncompressed RLE to RLE. + :return: binary mask (numpy 2D array) + """ + segm = ann['segmentation'] + if isinstance(segm, list): + # polygon -- a single object might consist of multiple parts + # we merge all parts into one mask rle code + rles = maskUtils.frPyObjects(segm, height, width) + rle = maskUtils.merge(rles) + elif isinstance(segm['counts'], list): + # uncompressed RLE + rle = maskUtils.frPyObjects(segm, height, width) + else: + # rle + rle = ann['segmentation'] + return rle + + def annToMask(self, ann, height, width): + """ + Convert annotation which can be polygons, uncompressed RLE, or RLE to binary mask. + :return: binary mask (numpy 2D array) + """ + rle = self.annToRLE(ann, height, width) + m = maskUtils.decode(rle) + return m + + +############################################################ +# COCO Evaluation +############################################################ + +def build_coco_results(dataset, image_ids, rois, class_ids, scores, masks): + """Arrange resutls to match COCO specs in http://cocodataset.org/#format + """ + # If no results, return an empty list + if rois is None: + return [] + + results = [] + for image_id in image_ids: + # Loop through detections + for i in range(rois.shape[0]): + class_id = class_ids[i] + score = scores[i] + bbox = np.around(rois[i], 1) + mask = masks[:, :, i] + + result = { + "image_id": image_id, + "category_id": dataset.get_source_class_id(class_id, "coco"), + "bbox": [bbox[1], bbox[0], bbox[3] - bbox[1], bbox[2] - bbox[0]], + "score": score, + "segmentation": maskUtils.encode(np.asfortranarray(mask)) + } + results.append(result) + return results + + +def evaluate_coco(model, dataset, coco, eval_type="bbox", limit=0, image_ids=None): + """Runs official COCO evaluation. + dataset: A Dataset object with valiadtion data + eval_type: "bbox" or "segm" for bounding box or segmentation evaluation + limit: if not 0, it's the number of images to use for evaluation + """ + # Pick COCO images from the dataset + image_ids = image_ids or dataset.image_ids + + # Limit to a subset + if limit: + image_ids = image_ids[:limit] + + # Get corresponding COCO image IDs. + coco_image_ids = [dataset.image_info[id]["id"] for id in image_ids] + + t_prediction = 0 + t_start = time.time() + + results = [] + for i, image_id in enumerate(image_ids): + # Load image + image = dataset.load_image(image_id) + + # Run detection + t = time.time() + r = model.detect([image], verbose=0)[0] + t_prediction += (time.time() - t) + + # Convert results to COCO format + # Cast masks to uint8 because COCO tools errors out on bool + image_results = build_coco_results(dataset, coco_image_ids[i:i + 1], + r["rois"], r["class_ids"], + r["scores"], + r["masks"].astype(np.uint8)) + results.extend(image_results) + + # Load results. This modifies results with additional attributes. + coco_results = coco.loadRes(results) + + # Evaluate + cocoEval = COCOeval(coco, coco_results, eval_type) + cocoEval.params.imgIds = coco_image_ids + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + + print("Prediction time: {}. Average {}/image".format( + t_prediction, t_prediction / len(image_ids))) + print("Total time: ", time.time() - t_start) + + +############################################################ +# Training +############################################################ + + +if __name__ == '__main__': + import argparse + + # Parse command line arguments + parser = argparse.ArgumentParser( + description='Train Mask R-CNN on MS COCO.') + parser.add_argument("command", + metavar="", + help="'train' or 'evaluate' on MS COCO") + parser.add_argument('--dataset', required=True, + metavar="/path/to/coco/", + help='Directory of the MS-COCO dataset') + parser.add_argument('--year', required=False, + default=DEFAULT_DATASET_YEAR, + metavar="", + help='Year of the MS-COCO dataset (2014 or 2017) (default=2014)') + parser.add_argument('--model', required=True, + metavar="/path/to/weights.h5", + help="Path to weights .h5 file or 'coco'") + parser.add_argument('--logs', required=False, + default=DEFAULT_LOGS_DIR, + metavar="/path/to/logs/", + help='Logs and checkpoints directory (default=logs/)') + parser.add_argument('--limit', required=False, + default=500, + metavar="", + help='Images to use for evaluation (default=500)') + parser.add_argument('--download', required=False, + default=False, + metavar="", + help='Automatically download and unzip MS-COCO files (default=False)', + type=bool) + args = parser.parse_args() + print("Command: ", args.command) + print("Model: ", args.model) + print("Dataset: ", args.dataset) + print("Year: ", args.year) + print("Logs: ", args.logs) + print("Auto Download: ", args.download) + + # Configurations + if args.command == "train": + config = CocoConfig() + else: + class InferenceConfig(CocoConfig): + # Set batch size to 1 since we'll be running inference on + # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU + GPU_COUNT = 1 + IMAGES_PER_GPU = 1 + DETECTION_MIN_CONFIDENCE = 0 + config = InferenceConfig() + config.display() + + # Create model + if args.command == "train": + model = modellib.MaskRCNN(mode="training", config=config, + model_dir=args.logs) + else: + model = modellib.MaskRCNN(mode="inference", config=config, + model_dir=args.logs) + + # Select weights file to load + if args.model.lower() == "coco": + model_path = COCO_MODEL_PATH + elif args.model.lower() == "last": + # Find last trained weights + model_path = model.find_last() + elif args.model.lower() == "imagenet": + # Start from ImageNet trained weights + model_path = model.get_imagenet_weights() + else: + model_path = args.model + + # Load weights + print("Loading weights ", model_path) + model.load_weights(model_path, by_name=True) + + # Train or evaluate + if args.command == "train": + # Training dataset. Use the training set and 35K from the + # validation set, as as in the Mask RCNN paper. + dataset_train = CocoDataset() + dataset_train.load_coco(args.dataset, "train", year=args.year, auto_download=args.download) + if args.year in '2014': + dataset_train.load_coco(args.dataset, "valminusminival", year=args.year, auto_download=args.download) + dataset_train.prepare() + + # Validation dataset + dataset_val = CocoDataset() + val_type = "val" if args.year in '2017' else "minival" + dataset_val.load_coco(args.dataset, val_type, year=args.year, auto_download=args.download) + dataset_val.prepare() + + # Image Augmentation + # Right/Left flip 50% of the time + augmentation = imgaug.augmenters.Fliplr(0.5) + + # *** This training schedule is an example. Update to your needs *** + + # Training - Stage 1 + print("Training network heads") + model.train(dataset_train, dataset_val, + learning_rate=config.LEARNING_RATE, + epochs=40, + layers='heads', + augmentation=augmentation) + + # Training - Stage 2 + # Finetune layers from ResNet stage 4 and up + print("Fine tune Resnet stage 4 and up") + model.train(dataset_train, dataset_val, + learning_rate=config.LEARNING_RATE, + epochs=120, + layers='4+', + augmentation=augmentation) + + # Training - Stage 3 + # Fine tune all layers + print("Fine tune all layers") + model.train(dataset_train, dataset_val, + learning_rate=config.LEARNING_RATE / 10, + epochs=160, + layers='all', + augmentation=augmentation) + + elif args.command == "evaluate": + # Validation dataset + dataset_val = CocoDataset() + val_type = "val" if args.year in '2017' else "minival" + coco = dataset_val.load_coco(args.dataset, val_type, year=args.year, return_coco=True, auto_download=args.download) + dataset_val.prepare() + print("Running COCO evaluation on {} images.".format(args.limit)) + evaluate_coco(model, dataset_val, coco, "bbox", limit=int(args.limit)) + else: + print("'{}' is not recognized. " + "Use 'train' or 'evaluate'".format(args.command)) diff --git a/mask_rcnn/tf_servable/config.py b/mask_rcnn/tf_servable/config.py new file mode 100644 index 00000000..90286026 --- /dev/null +++ b/mask_rcnn/tf_servable/config.py @@ -0,0 +1,67 @@ +import numpy as np + + +class mask_config(): + def __init__(self, NUMBER_OF_CLASSES): + self.NAME = "tags" + self.IMAGES_PER_GPU = 2 + self.NUM_CLASSES = 1 + NUMBER_OF_CLASSES # Background + tags + self.STEPS_PER_EPOCH = 100 + self.DETECTION_MIN_CONFIDENCE = 0.9 + self.GPU_COUNT = 1 + self.IMAGES_PER_GPU = 1 + self.NAME = None # Override in sub-classes + self.GPU_COUNT = 1 + self.IMAGES_PER_GPU = 1 + self.STEPS_PER_EPOCH = 1000 + self.VALIDATION_STEPS = 50 + self.BACKBONE = "resnet101" + self.COMPUTE_BACKBONE_SHAPE = None + self.BACKBONE_STRIDES = [4, 8, 16, 32, 64] + self.FPN_CLASSIF_FC_LAYERS_SIZE = 1024 + self.TOP_DOWN_PYRAMID_SIZE = 256 + self.RPN_ANCHOR_SCALES = (32, 64, 128, 256, 512) + self.RPN_ANCHOR_RATIOS = [0.5, 1, 2] + self.RPN_ANCHOR_STRIDE = 1 + self.RPN_NMS_THRESHOLD = 0.7 + self.RPN_TRAIN_ANCHORS_PER_IMAGE = 256 + self.POST_NMS_ROIS_TRAINING = 2000 + self.POST_NMS_ROIS_INFERENCE = 1000 + self.USE_MINI_MASK = True + self.MINI_MASK_SHAPE = (56, 56) # (height, width) of the mini-mask + self.IMAGE_RESIZE_MODE = "square" + self.IMAGE_MIN_DIM = 800 + self.IMAGE_MAX_DIM = 1024 + self.IMAGE_MIN_SCALE = 0 + self.MEAN_PIXEL = np.array([123.7, 116.8, 103.9]) + self.TRAIN_ROIS_PER_IMAGE = 200 + self.ROI_POSITIVE_RATIO = 0.33 + self.POOL_SIZE = 7 + self.MASK_POOL_SIZE = 14 + self.MASK_SHAPE = [28, 28] + self.MAX_GT_INSTANCES = 100 + self.RPN_BBOX_STD_DEV = np.array([0.1, 0.1, 0.2, 0.2]) + self.BBOX_STD_DEV = np.array([0.1, 0.1, 0.2, 0.2]) + self.DETECTION_MAX_INSTANCES = 100 + self.DETECTION_MIN_CONFIDENCE = 0.7 + self.DETECTION_NMS_THRESHOLD = 0.3 + self.LEARNING_RATE = 0.001 + self.LEARNING_MOMENTUM = 0.9 + self.WEIGHT_DECAY = 0.0001 + self.LOSS_WEIGHTS = {"rpn_class_loss": 1., "rpn_bbox_loss": 1., "mrcnn_class_loss": 1., "mrcnn_bbox_loss": 1., + "mrcnn_mask_loss": 1.} + self.USE_RPN_ROIS = True + self.TRAIN_BN = False # Defaulting to False since batch size is often small + self.GRADIENT_CLIP_NORM = 5.0 + + self.BATCH_SIZE = self.IMAGES_PER_GPU * self.GPU_COUNT + + # Input image size + if self.IMAGE_RESIZE_MODE == "crop": + self.IMAGE_SHAPE = np.array([self.IMAGE_MIN_DIM, self.IMAGE_MIN_DIM, 3]) + else: + self.IMAGE_SHAPE = np.array([self.IMAGE_MAX_DIM, self.IMAGE_MAX_DIM, 3]) + + # Image meta data length + # See compose_image_meta() for details + self.IMAGE_META_SIZE = 1 + 3 + 3 + 4 + 1 + self.NUM_CLASSES diff --git a/mask_rcnn/tf_servable/frozen_model/.gitkeep b/mask_rcnn/tf_servable/frozen_model/.gitkeep new file mode 100644 index 00000000..74c20143 --- /dev/null +++ b/mask_rcnn/tf_servable/frozen_model/.gitkeep @@ -0,0 +1 @@ +DUMMY \ No newline at end of file diff --git a/mask_rcnn/tf_servable/inferencing/config.py b/mask_rcnn/tf_servable/inferencing/config.py new file mode 100644 index 00000000..5bffb33d --- /dev/null +++ b/mask_rcnn/tf_servable/inferencing/config.py @@ -0,0 +1,236 @@ +""" +Mask R-CNN +Base Configurations class. + +Copyright (c) 2017 Matterport, Inc. +Licensed under the MIT License (see LICENSE for details) +Written by Waleed Abdulla +""" + +import numpy as np + + +# Base Configuration Class +# Don't use this class directly. Instead, sub-class it and override +# the configurations you need to change. + +class Config(object): + """Base configuration class. For custom configurations, create a + sub-class that inherits from this one and override properties + that need to be changed. + """ + # Name the configurations. For example, 'COCO', 'Experiment 3', ...etc. + # Useful if your code needs to do things differently depending on which + # experiment is running. + NAME = None # Override in sub-classes + + # NUMBER OF GPUs to use. When using only a CPU, this needs to be set to 1. + GPU_COUNT = 1 + + # Number of images to train with on each GPU. A 12GB GPU can typically + # handle 2 images of 1024x1024px. + # Adjust based on your GPU memory and image sizes. Use the highest + # number that your GPU can handle for best performance. + IMAGES_PER_GPU = 2 + + # Number of training steps per epoch + # This doesn't need to match the size of the training set. Tensorboard + # updates are saved at the end of each epoch, so setting this to a + # smaller number means getting more frequent TensorBoard updates. + # Validation stats are also calculated at each epoch end and they + # might take a while, so don't set this too small to avoid spending + # a lot of time on validation stats. + STEPS_PER_EPOCH = 1000 + + # Number of validation steps to run at the end of every training epoch. + # A bigger number improves accuracy of validation stats, but slows + # down the training. + VALIDATION_STEPS = 50 + + # Backbone network architecture + # Supported values are: resnet50, resnet101. + # You can also provide a callable that should have the signature + # of model.resnet_graph. If you do so, you need to supply a callable + # to COMPUTE_BACKBONE_SHAPE as well + BACKBONE = "resnet101" + + # Only useful if you supply a callable to BACKBONE. Should compute + # the shape of each layer of the FPN Pyramid. + # See model.compute_backbone_shapes + COMPUTE_BACKBONE_SHAPE = None + + # The strides of each layer of the FPN Pyramid. These values + # are based on a Resnet101 backbone. + BACKBONE_STRIDES = [4, 8, 16, 32, 64] + + # Size of the fully-connected layers in the classification graph + FPN_CLASSIF_FC_LAYERS_SIZE = 1024 + + # Size of the top-down layers used to build the feature pyramid + TOP_DOWN_PYRAMID_SIZE = 256 + + # Number of classification classes (including background) + NUM_CLASSES = 1 # Override in sub-classes + + # Length of square anchor side in pixels + RPN_ANCHOR_SCALES = (32, 64, 128, 256, 512) + + # Ratios of anchors at each cell (width/height) + # A value of 1 represents a square anchor, and 0.5 is a wide anchor + RPN_ANCHOR_RATIOS = [0.5, 1, 2] + + # Anchor stride + # If 1 then anchors are created for each cell in the backbone feature map. + # If 2, then anchors are created for every other cell, and so on. + RPN_ANCHOR_STRIDE = 1 + + # Non-max suppression threshold to filter RPN proposals. + # You can increase this during training to generate more propsals. + RPN_NMS_THRESHOLD = 0.7 + + # How many anchors per image to use for RPN training + RPN_TRAIN_ANCHORS_PER_IMAGE = 256 + + # ROIs kept after tf.nn.top_k and before non-maximum suppression + PRE_NMS_LIMIT = 6000 + + # ROIs kept after non-maximum suppression (training and inference) + POST_NMS_ROIS_TRAINING = 2000 + POST_NMS_ROIS_INFERENCE = 1000 + + # If enabled, resizes instance masks to a smaller size to reduce + # memory load. Recommended when using high-resolution images. + USE_MINI_MASK = False + MINI_MASK_SHAPE = (56, 56) # (height, width) of the mini-mask + + # Input image resizing + # Generally, use the "square" resizing mode for training and predicting + # and it should work well in most cases. In this mode, images are scaled + # up such that the small side is = IMAGE_MIN_DIM, but ensuring that the + # scaling doesn't make the long side > IMAGE_MAX_DIM. Then the image is + # padded with zeros to make it a square so multiple images can be put + # in one batch. + # Available resizing modes: + # none: No resizing or padding. Return the image unchanged. + # square: Resize and pad with zeros to get a square image + # of size [max_dim, max_dim]. + # pad64: Pads width and height with zeros to make them multiples of 64. + # If IMAGE_MIN_DIM or IMAGE_MIN_SCALE are not None, then it scales + # up before padding. IMAGE_MAX_DIM is ignored in this mode. + # The multiple of 64 is needed to ensure smooth scaling of feature + # maps up and down the 6 levels of the FPN pyramid (2**6=64). + # crop: Picks random crops from the image. First, scales the image based + # on IMAGE_MIN_DIM and IMAGE_MIN_SCALE, then picks a random crop of + # size IMAGE_MIN_DIM x IMAGE_MIN_DIM. Can be used in training only. + # IMAGE_MAX_DIM is not used in this mode. + IMAGE_RESIZE_MODE = "square" + IMAGE_MIN_DIM = 800 + IMAGE_MAX_DIM = 1024 + # Minimum scaling ratio. Checked after MIN_IMAGE_DIM and can force further + # up scaling. For example, if set to 2 then images are scaled up to double + # the width and height, or more, even if MIN_IMAGE_DIM doesn't require it. + # However, in 'square' mode, it can be overruled by IMAGE_MAX_DIM. + IMAGE_MIN_SCALE = 0 + # Number of color channels per image. RGB = 3, grayscale = 1, RGB-D = 4 + # Changing this requires other changes in the code. See the WIKI for more + # details: https://github.com/matterport/Mask_RCNN/wiki + IMAGE_CHANNEL_COUNT = 3 + + # Image mean (RGB) + MEAN_PIXEL = np.array([123.7, 116.8, 103.9]) + + # Number of ROIs per image to feed to classifier/mask heads + # The Mask RCNN paper uses 512 but often the RPN doesn't generate + # enough positive proposals to fill this and keep a positive:negative + # ratio of 1:3. You can increase the number of proposals by adjusting + # the RPN NMS threshold. + TRAIN_ROIS_PER_IMAGE = 200 + + # Percent of positive ROIs used to train classifier/mask heads + ROI_POSITIVE_RATIO = 0.33 + + # Pooled ROIs + POOL_SIZE = 7 + MASK_POOL_SIZE = 14 + + # Shape of output mask + # To change this you also need to change the neural network mask branch + MASK_SHAPE = [28, 28] + + # Maximum number of ground truth instances to use in one image + MAX_GT_INSTANCES = 100 + + # Bounding box refinement standard deviation for RPN and final detections. + RPN_BBOX_STD_DEV = np.array([0.1, 0.1, 0.2, 0.2]) + BBOX_STD_DEV = np.array([0.1, 0.1, 0.2, 0.2]) + + # Max number of final detections + DETECTION_MAX_INSTANCES = 100 + + # Minimum probability value to accept a detected instance + # ROIs below this threshold are skipped + DETECTION_MIN_CONFIDENCE = 0.7 + + # Non-maximum suppression threshold for detection + DETECTION_NMS_THRESHOLD = 0.3 + + # Learning rate and momentum + # The Mask RCNN paper uses lr=0.02, but on TensorFlow it causes + # weights to explode. Likely due to differences in optimizer + # implementation. + LEARNING_RATE = 0.001 + LEARNING_MOMENTUM = 0.9 + + # Weight decay regularization + WEIGHT_DECAY = 0.0001 + + # Loss weights for more precise optimization. + # Can be used for R-CNN training setup. + LOSS_WEIGHTS = { + "rpn_class_loss": 1., + "rpn_bbox_loss": 1., + "mrcnn_class_loss": 1., + "mrcnn_bbox_loss": 1., + "mrcnn_mask_loss": 1. + } + + # Use RPN ROIs or externally generated ROIs for training + # Keep this True for most situations. Set to False if you want to train + # the head branches on ROI generated by code rather than the ROIs from + # the RPN. For example, to debug the classifier head without having to + # train the RPN. + USE_RPN_ROIS = True + + # Train or freeze batch normalization layers + # None: Train BN layers. This is the normal mode + # False: Freeze BN layers. Good when using a small batch size + # True: (don't use). Set layer in training mode even when predicting + TRAIN_BN = False # Defaulting to False since batch size is often small + + # Gradient norm clipping + GRADIENT_CLIP_NORM = 5.0 + + def __init__(self): + """Set values of computed attributes.""" + # Effective batch size + self.BATCH_SIZE = self.IMAGES_PER_GPU * self.GPU_COUNT + + # Input image size + if self.IMAGE_RESIZE_MODE == "crop": + self.IMAGE_SHAPE = np.array([self.IMAGE_MIN_DIM, self.IMAGE_MIN_DIM, + self.IMAGE_CHANNEL_COUNT]) + else: + self.IMAGE_SHAPE = np.array([self.IMAGE_MAX_DIM, self.IMAGE_MAX_DIM, + self.IMAGE_CHANNEL_COUNT]) + + # Image meta data length + # See compose_image_meta() for details + self.IMAGE_META_SIZE = 1 + 3 + 3 + 4 + 1 + self.NUM_CLASSES + + def display(self): + """Display Configuration values.""" + print("\nConfigurations:") + for a in dir(self): + if not a.startswith("__") and not callable(getattr(self, a)): + print("{:30} {}".format(a, getattr(self, a))) + print("\n") diff --git a/mask_rcnn/tf_servable/inferencing/saved_model_config.py b/mask_rcnn/tf_servable/inferencing/saved_model_config.py new file mode 100644 index 00000000..e8b9caa7 --- /dev/null +++ b/mask_rcnn/tf_servable/inferencing/saved_model_config.py @@ -0,0 +1,50 @@ +# Your Inference Config Class +# Replace your own config +# MY_INFERENCE_CONFIG = YOUR_CONFIG_CLASS +# import coco +# class InferenceConfig(coco.CocoConfig): +# GPU_COUNT = 1 +# IMAGES_PER_GPU = 1 +# coco_config = InferenceConfig() +import config +class PPConfig(config.Config): + """Configuration for training on the toy dataset. + Derives from the base Config class and overrides some values. + """ + # Give the configuration a recognizable name + NAME = "pointless_package" + + # We use a GPU with 12GB memory, which can fit two images. + # Adjust down if you use a smaller GPU. + IMAGES_PER_GPU = 1 + + # Skip detections with < 90% confidence + DETECTION_MIN_CONFIDENCE = 0.75 + +MY_INFERENCE_CONFIG = PPConfig() + + +# Tensorflow Model server variable +ADDRESS = 'localhost' +PORT_NO_GRPC = 8500 +PORT_NO_RESTAPI = 8501 +MODEL_NAME = 'mask' +REST_API_URL = "http://%s:%s/v1/models/%s:predict" % (ADDRESS, PORT_NO_RESTAPI, MODEL_NAME) + + +# TF variable name +OUTPUT_DETECTION = 'mrcnn_detection/Reshape_1' +OUTPUT_CLASS = 'mrcnn_class/Reshape_1' +OUTPUT_BBOX = 'mrcnn_bbox/Reshape' +OUTPUT_MASK = 'mrcnn_mask/Reshape_1' +INPUT_IMAGE = 'input_image' +INPUT_IMAGE_META = 'input_image_meta' +INPUT_ANCHORS = 'input_anchors' +OUTPUT_NAME = 'predict_images' + + +# Signature name +SIGNATURE_NAME = 'serving_default' + +# GRPC config +GRPC_MAX_RECEIVE_MESSAGE_LENGTH = 4096 * 4096 * 3 # Max LENGTH the GRPC should handle diff --git a/mask_rcnn/tf_servable/inferencing/saved_model_inference.py b/mask_rcnn/tf_servable/inferencing/saved_model_inference.py new file mode 100644 index 00000000..ffd9988c --- /dev/null +++ b/mask_rcnn/tf_servable/inferencing/saved_model_inference.py @@ -0,0 +1,138 @@ +import cv2, grpc +from tensorflow_serving.apis import prediction_service_pb2_grpc +from tensorflow_serving.apis import predict_pb2 +import numpy as np +import tensorflow as tf +import saved_model_config +from saved_model_preprocess import ForwardModel +import requests +import json +from visualize import display_images +import visualize as visualize + +host = saved_model_config.ADDRESS +PORT_GRPC = saved_model_config.PORT_NO_GRPC +RESTAPI_URL = saved_model_config.REST_API_URL + +channel = grpc.insecure_channel(str(host) + ':' + str(PORT_GRPC), options=[('grpc.max_receive_message_length', saved_model_config.GRPC_MAX_RECEIVE_MESSAGE_LENGTH)]) + +stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) + + +request = predict_pb2.PredictRequest() +request.model_spec.name = saved_model_config.MODEL_NAME +request.model_spec.signature_name = saved_model_config.SIGNATURE_NAME + +model_config = saved_model_config.MY_INFERENCE_CONFIG +preprocess_obj = ForwardModel(model_config) + + +def detect_mask_single_image_using_grpc(image): + images = np.expand_dims(image, axis=0) + molded_images, image_metas, windows = preprocess_obj.mold_inputs(images) + molded_images = molded_images.astype(np.float32) + image_metas = image_metas.astype(np.float32) + # Validate image sizes + # All images in a batch MUST be of the same size + image_shape = molded_images[0].shape + for g in molded_images[1:]: + assert g.shape == image_shape, \ + "After resizing, all images must have the same size. Check IMAGE_RESIZE_MODE and image sizes." + + # Anchors + anchors = preprocess_obj.get_anchors(image_shape) + anchors = np.broadcast_to(anchors, (images.shape[0],) + anchors.shape) + + request.inputs[saved_model_config.INPUT_IMAGE].CopyFrom( + tf.contrib.util.make_tensor_proto(molded_images, shape=molded_images.shape)) + request.inputs[saved_model_config.INPUT_IMAGE_META].CopyFrom( + tf.contrib.util.make_tensor_proto(image_metas, shape=image_metas.shape)) + request.inputs[saved_model_config.INPUT_ANCHORS].CopyFrom( + tf.contrib.util.make_tensor_proto(anchors, shape=anchors.shape)) + + result = stub.Predict(request, 60.) + result_dict = preprocess_obj.result_to_dict(images, molded_images, windows, result)[0] + return result_dict + + +def detect_mask_single_image_using_restapi(image): + images = np.expand_dims(image, axis=0) + molded_images, image_metas, windows = preprocess_obj.mold_inputs(images) + + molded_images = molded_images.astype(np.float32) + + image_shape = molded_images[0].shape + + for g in molded_images[1:]: + assert g.shape == image_shape, \ + "After resizing, all images must have the same size. Check IMAGE_RESIZE_MODE and image sizes." + + anchors = preprocess_obj.get_anchors(image_shape) + anchors = np.broadcast_to(anchors, (images.shape[0],) + anchors.shape) + + # response body format row wise. + data = {'signature_name': saved_model_config.SIGNATURE_NAME, + 'instances': [{saved_model_config.INPUT_IMAGE: molded_images[0].tolist(), + saved_model_config.INPUT_IMAGE_META: image_metas[0].tolist(), + saved_model_config.INPUT_ANCHORS: anchors[0].tolist()}]} + + response = requests.post(RESTAPI_URL, data=json.dumps(data), headers={"content-type":"application/json"}) + result = json.loads(response.text) + result = result['predictions'][0] + + result_dict = preprocess_obj.result_to_dict(images, molded_images, windows, result, is_restapi=True)[0] + return result_dict + + +if __name__ == '__main__': + import argparse + import os + parser = argparse.ArgumentParser() + parser.add_argument('-p', '--path', help='Path to Image', required=True) + parser.add_argument('-t', '--type', help='Type of call [restapi, grpc]', default='restapi') + args = vars(parser.parse_args()) + image_path = args['path'] + call_type = args['type'] + + if not os.path.exists(image_path): + print(image_path, " -- Does not exist") + exit() + + image = cv2.imread(image_path) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + if image is None: + print("Image path is not proper") + exit() + + if call_type == 'restapi': + result = detect_mask_single_image_using_restapi(image) + else: + result = detect_mask_single_image_using_grpc(image) + + print(result) + r = result + + N = r['rois'].shape[0] + class_ids = r['class'] + masks = r['mask'] + + CLASS_NAMES = ['BG', 'outerbox', 'innerbox', 'item_sq', + 'item_rect', 'item_rect_slim', 'item_circ'] + + class_names = np.asarray(CLASS_NAMES) + ITEM_NAMES = CLASS_NAMES[3:] + # area_occupation = [masks[:, :, i].sum() for i in range(N)] + area_occupation = masks.sum(axis=0).sum(axis=0) + map_items_to_area = list(map( + lambda x, y: x+': '+str(y), class_names[class_ids], area_occupation)) + print("{}: (ITEM: AREA) ->".format('IMAGE'), map_items_to_area, "{}".format( + "" if set(ITEM_NAMES) & set(class_names[class_ids]) else "-> NO ITEMS FOUND!")) + + visualize.display_instances(image, r['rois'], r['mask'], r['class'], + CLASS_NAMES, r['scores'], + title="Predictions") + + print("*" * 60) + print("RESULTS:") + print(result) + print("*" * 60) diff --git a/mask_rcnn/tf_servable/inferencing/saved_model_preprocess.py b/mask_rcnn/tf_servable/inferencing/saved_model_preprocess.py new file mode 100644 index 00000000..d43ddfb5 --- /dev/null +++ b/mask_rcnn/tf_servable/inferencing/saved_model_preprocess.py @@ -0,0 +1,243 @@ +import saved_model_config +import saved_model_utils +import numpy as np +import math + + +def compose_image_meta(image_id, original_image_shape, image_shape, + window, scale, active_class_ids): + """Takes attributes of an image and puts them in one 1D array. + image_id: An int ID of the image. Useful for debugging. + original_image_shape: [H, W, C] before resizing or padding. + image_shape: [H, W, C] after resizing and padding + window: (y1, x1, y2, x2) in pixels. The area of the image where the real + image is (excluding the padding) + scale: The scaling factor applied to the original image (float32) + active_class_ids: List of class_ids available in the dataset from which + the image came. Useful if training on images from multiple segmentation_datasets + where not all classes are present in all segmentation_datasets. + """ + meta = np.array( + [image_id] + # size=1 + list(original_image_shape) + # size=3 + list(image_shape) + # size=3 + list(window) + # size=4 (y1, x1, y2, x2) in image cooredinates + [scale] + # size=1 + list(active_class_ids) # size=num_classes + ) + return meta + + +def mold_image(images, config): + """Expects an RGB image (or array of images) and subtracts + the mean pixel and converts it to float. Expects image + colors in RGB order. + """ + return images.astype(np.float32) - config.MEAN_PIXEL + + +def compute_backbone_shapes(config, image_shape): + """Computes the width and height of each stage of the backbone network. + + Returns: + [N, (height, width)]. Where N is the number of stages + """ + if callable(config.BACKBONE): + return config.COMPUTE_BACKBONE_SHAPE(image_shape) + + # Currently supports ResNet only + assert config.BACKBONE in ["resnet50", "resnet101"] + return np.array( + [[int(math.ceil(image_shape[0] / stride)), + int(math.ceil(image_shape[1] / stride))] + for stride in config.BACKBONE_STRIDES]) + + +class ForwardModel: + def __init__(self, config): + self.config = config + self.outputs = { + 'detection': 'mrcnn_detection/Reshape_1', + 'class': 'mrcnn_class/Reshape_1', + 'box': 'mrcnn_bbox/Reshape', + 'mask': 'mrcnn_mask/Reshape_1'} + + # self.build_outputs() + + def mold_inputs(self, images): + """Takes a list of images and modifies them to the format expected + as an input to the neural network. + images: List of image matrices [height,width,depth]. Images can have + different sizes. + + Returns 3 Numpy matrices: + molded_images: [N, h, w, 3]. Images resized and normalized. + image_metas: [N, length of meta data]. Details about each image. + windows: [N, (y1, x1, y2, x2)]. The portion of the image that has the + original image (padding excluded). + """ + molded_images = [] + image_metas = [] + windows = [] + for image in images: + # Resize image + # TODO: move resizing to mold_image() + molded_image, window, scale, padding, crop = saved_model_utils.resize_image( + image, + min_dim=self.config.IMAGE_MIN_DIM, + min_scale=self.config.IMAGE_MIN_SCALE, + max_dim=self.config.IMAGE_MAX_DIM, + mode=self.config.IMAGE_RESIZE_MODE) + molded_image = mold_image(molded_image, self.config) + # Build image_meta + image_meta = compose_image_meta( + 0, image.shape, molded_image.shape, window, scale, + np.zeros([self.config.NUM_CLASSES], dtype=np.int32)) + # Append + molded_images.append(molded_image) + windows.append(window) + image_metas.append(image_meta) + # Pack into arrays + molded_images = np.stack(molded_images) + image_metas = np.stack(image_metas) + windows = np.stack(windows) + return molded_images, image_metas, windows + + def get_anchors(self, image_shape): + """Returns anchor pyramid for the given image size.""" + backbone_shapes = compute_backbone_shapes(self.config, image_shape) + # Cache anchors and reuse if image shape is the same + if not hasattr(self, "_anchor_cache"): + self._anchor_cache = {} + if not tuple(image_shape) in self._anchor_cache: + # Generate Anchors + a = saved_model_utils.generate_pyramid_anchors( + self.config.RPN_ANCHOR_SCALES, + self.config.RPN_ANCHOR_RATIOS, + backbone_shapes, + self.config.BACKBONE_STRIDES, + self.config.RPN_ANCHOR_STRIDE) + # Keep a copy of the latest anchors in pixel coordinates because + # it's used in inspect_model notebooks. + # TODO: Remove this after the notebook are refactored to not use it + self.anchors = a + # Normalize coordinates + self._anchor_cache[tuple(image_shape)] = saved_model_utils.norm_boxes(a, image_shape[:2]) + return self._anchor_cache[tuple(image_shape)] + + def unmold_detections(self, detections, mrcnn_mask, original_image_shape, + image_shape, window): + """Reformats the detections of one image from the format of the neural + network output to a format suitable for use in the rest of the + application. + + detections: [N, (y1, x1, y2, x2, class_id, score)] in normalized coordinates + mrcnn_mask: [N, height, width, num_classes] + original_image_shape: [H, W, C] Original image shape before resizing + image_shape: [H, W, C] Shape of the image after resizing and padding + window: [y1, x1, y2, x2] Pixel coordinates of box in the image where the real + image is excluding the padding. + + Returns: + boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels + class_ids: [N] Integer class IDs for each bounding box + scores: [N] Float probability scores of the class_id + masks: [height, width, num_instances] Instance masks + """ + # How many detections do we have? + # Detections array is padded with zeros. Find the first class_id == 0. + zero_ix = np.where(detections[:, 4] == 0)[0] + N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0] + + # Extract boxes, class_ids, scores, and class-specific masks + boxes = detections[:N, :4] + class_ids = detections[:N, 4].astype(np.int32) + scores = detections[:N, 5] + masks = mrcnn_mask[np.arange(N), :, :, class_ids] + + # Translate normalized coordinates in the resized image to pixel + # coordinates in the original image before resizing + window = saved_model_utils.norm_boxes(window, image_shape[:2]) + wy1, wx1, wy2, wx2 = window + shift = np.array([wy1, wx1, wy1, wx1]) + wh = wy2 - wy1 # window height + ww = wx2 - wx1 # window width + scale = np.array([wh, ww, wh, ww]) + # Convert boxes to normalized coordinates on the window + boxes = np.divide(boxes - shift, scale) + # Convert boxes to pixel coordinates on the original image + boxes = saved_model_utils.denorm_boxes(boxes, original_image_shape[:2]) + + # Filter out detections with zero area. Happens in early training when + # network weights are still random + exclude_ix = np.where( + (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0] + if exclude_ix.shape[0] > 0: + boxes = np.delete(boxes, exclude_ix, axis=0) + class_ids = np.delete(class_ids, exclude_ix, axis=0) + scores = np.delete(scores, exclude_ix, axis=0) + masks = np.delete(masks, exclude_ix, axis=0) + N = class_ids.shape[0] + + # Resize masks to original image size and set boundary threshold. + full_masks = [] + for i in range(N): + # Convert neural network mask to full size mask + full_mask = saved_model_utils.unmold_mask(masks[i], boxes[i], original_image_shape) + full_masks.append(full_mask) + full_masks = np.stack(full_masks, axis=-1)\ + if full_masks else np.empty(original_image_shape[:2] + (0,)) + + return boxes, class_ids, scores, full_masks + + def format_output(self, result_dict): + mask_shape = result_dict.outputs[saved_model_config.OUTPUT_MASK].tensor_shape.dim + mask_shape = tuple(d.size for d in mask_shape) + mask = np.array(result_dict.outputs[saved_model_config.OUTPUT_MASK].float_val) + mask = np.reshape(mask, mask_shape) + + detection_shape = result_dict.outputs[saved_model_config.OUTPUT_DETECTION].tensor_shape.dim + detection_shape = tuple(d.size for d in detection_shape) + detection = np.array(result_dict.outputs[saved_model_config.OUTPUT_DETECTION].float_val) + detection = np.reshape(detection, detection_shape) + + result_dict = {'detection': detection, 'mask': mask} + + return result_dict + + def format_restapi_output(self, result_dict): + mask = result_dict[saved_model_config.OUTPUT_MASK] + mask = np.array(mask) + mask = np.expand_dims(mask, axis=0) + + detection = result_dict[saved_model_config.OUTPUT_DETECTION] + detection = np.array(detection) + detection = np.expand_dims(detection, axis=0) + + result_dict = {'detection': detection, 'mask': mask} + return result_dict + + def result_to_dict(self, images, molded_images, windows, result_dict, is_restapi=False): + if is_restapi: + result_dict = self.format_restapi_output(result_dict) + else: + result_dict = self.format_output(result_dict) + results = [] + for i, image in enumerate(images): + # print('detection len',len(result_dict['detection'])) + # print('mask len ',len(result_dict['mask'])) + final_rois, final_class_ids, final_scores, final_masks = \ + self.unmold_detections(result_dict['detection'][i], result_dict['mask'][i], + image.shape, molded_images[i].shape, + windows[i]) + results.append({ + "rois": final_rois, + "class": final_class_ids, + "scores": final_scores, + "mask": final_masks, + }) + # print('rois:', final_rois.shape) + # print('class:', final_class_ids.shape) + # print('scores:', final_scores.shape) + # print('final mask shaoe:', final_masks.shape) + return results \ No newline at end of file diff --git a/mask_rcnn/tf_servable/inferencing/saved_model_utils.py b/mask_rcnn/tf_servable/inferencing/saved_model_utils.py new file mode 100644 index 00000000..a6823fea --- /dev/null +++ b/mask_rcnn/tf_servable/inferencing/saved_model_utils.py @@ -0,0 +1,879 @@ +import random +import cv2 +import numpy as np +import tensorflow as tf +import scipy +import skimage.color +import skimage.transform +import urllib.request +import shutil +import warnings + + +COCO_MODEL_URL = "https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5" + + +############################################################ +# Bounding Boxes +############################################################ + +def extract_bboxes(mask): + """Compute bounding boxes from masks. + mask: [height, width, num_instances]. Mask pixels are either 1 or 0. + + Returns: bbox array [num_instances, (y1, x1, y2, x2)]. + """ + boxes = np.zeros([mask.shape[-1], 4], dtype=np.int32) + for i in range(mask.shape[-1]): + m = mask[:, :, i] + # Bounding box. + horizontal_indicies = np.where(np.any(m, axis=0))[0] + vertical_indicies = np.where(np.any(m, axis=1))[0] + if horizontal_indicies.shape[0]: + x1, x2 = horizontal_indicies[[0, -1]] + y1, y2 = vertical_indicies[[0, -1]] + # x2 and y2 should not be part of the box. Increment by 1. + x2 += 1 + y2 += 1 + else: + # No mask for this instance. Might happen due to + # resizing or cropping. Set bbox to zeros + x1, x2, y1, y2 = 0, 0, 0, 0 + boxes[i] = np.array([y1, x1, y2, x2]) + return boxes.astype(np.int32) + + +def compute_iou(box, boxes, box_area, boxes_area): + """Calculates IoU of the given box with the array of the given boxes. + box: 1D vector [y1, x1, y2, x2] + boxes: [boxes_count, (y1, x1, y2, x2)] + box_area: float. the area of 'box' + boxes_area: array of length boxes_count. + + Note: the areas are passed in rather than calculated here for + efficiency. Calculate once in the caller to avoid duplicate work. + """ + # Calculate intersection areas + y1 = np.maximum(box[0], boxes[:, 0]) + y2 = np.minimum(box[2], boxes[:, 2]) + x1 = np.maximum(box[1], boxes[:, 1]) + x2 = np.minimum(box[3], boxes[:, 3]) + intersection = np.maximum(x2 - x1, 0) * np.maximum(y2 - y1, 0) + union = box_area + boxes_area[:] - intersection[:] + iou = intersection / union + return iou + + +def compute_overlaps(boxes1, boxes2): + """Computes IoU overlaps between two sets of boxes. + boxes1, boxes2: [N, (y1, x1, y2, x2)]. + + For better performance, pass the largest set first and the smaller second. + """ + # Areas of anchors and GT boxes + area1 = (boxes1[:, 2] - boxes1[:, 0]) * (boxes1[:, 3] - boxes1[:, 1]) + area2 = (boxes2[:, 2] - boxes2[:, 0]) * (boxes2[:, 3] - boxes2[:, 1]) + + # Compute overlaps to generate matrix [boxes1 count, boxes2 count] + # Each cell contains the IoU value. + overlaps = np.zeros((boxes1.shape[0], boxes2.shape[0])) + for i in range(overlaps.shape[1]): + box2 = boxes2[i] + overlaps[:, i] = compute_iou(box2, boxes1, area2[i], area1) + return overlaps + + +def compute_overlaps_masks(masks1, masks2): + """Computes IoU overlaps between two sets of masks. + masks1, masks2: [Height, Width, instances] + """ + + # If either set of masks is empty return empty result + if masks1.shape[0] == 0 or masks2.shape[0] == 0: + return np.zeros((masks1.shape[0], masks2.shape[-1])) + # flatten masks and compute their areas + masks1 = np.reshape(masks1 > .5, (-1, masks1.shape[-1])).astype(np.float32) + masks2 = np.reshape(masks2 > .5, (-1, masks2.shape[-1])).astype(np.float32) + area1 = np.sum(masks1, axis=0) + area2 = np.sum(masks2, axis=0) + + # intersections and union + intersections = np.dot(masks1.T, masks2) + union = area1[:, None] + area2[None, :] - intersections + overlaps = intersections / union + + return overlaps + + +def non_max_suppression(boxes, scores, threshold): + """Performs non-maximum suppression and returns indices of kept boxes. + boxes: [N, (y1, x1, y2, x2)]. Notice that (y2, x2) lays outside the box. + scores: 1-D array of box scores. + threshold: Float. IoU threshold to use for filtering. + """ + assert boxes.shape[0] > 0 + if boxes.dtype.kind != "f": + boxes = boxes.astype(np.float32) + + # Compute box areas + y1 = boxes[:, 0] + x1 = boxes[:, 1] + y2 = boxes[:, 2] + x2 = boxes[:, 3] + area = (y2 - y1) * (x2 - x1) + + # Get indicies of boxes sorted by scores (highest first) + ixs = scores.argsort()[::-1] + + pick = [] + while len(ixs) > 0: + # Pick top box and add its index to the list + i = ixs[0] + pick.append(i) + # Compute IoU of the picked box with the rest + iou = compute_iou(boxes[i], boxes[ixs[1:]], area[i], area[ixs[1:]]) + # Identify boxes with IoU over the threshold. This + # returns indices into ixs[1:], so add 1 to get + # indices into ixs. + remove_ixs = np.where(iou > threshold)[0] + 1 + # Remove indices of the picked and overlapped boxes. + ixs = np.delete(ixs, remove_ixs) + ixs = np.delete(ixs, 0) + return np.array(pick, dtype=np.int32) + + +def apply_box_deltas(boxes, deltas): + """Applies the given deltas to the given boxes. + boxes: [N, (y1, x1, y2, x2)]. Note that (y2, x2) is outside the box. + deltas: [N, (dy, dx, log(dh), log(dw))] + """ + boxes = boxes.astype(np.float32) + # Convert to y, x, h, w + height = boxes[:, 2] - boxes[:, 0] + width = boxes[:, 3] - boxes[:, 1] + center_y = boxes[:, 0] + 0.5 * height + center_x = boxes[:, 1] + 0.5 * width + # Apply deltas + center_y += deltas[:, 0] * height + center_x += deltas[:, 1] * width + height *= np.exp(deltas[:, 2]) + width *= np.exp(deltas[:, 3]) + # Convert back to y1, x1, y2, x2 + y1 = center_y - 0.5 * height + x1 = center_x - 0.5 * width + y2 = y1 + height + x2 = x1 + width + return np.stack([y1, x1, y2, x2], axis=1) + + +def box_refinement_graph(box, gt_box): + """Compute refinement needed to transform box to gt_box. + box and gt_box are [N, (y1, x1, y2, x2)] + """ + box = tf.cast(box, tf.float32) + gt_box = tf.cast(gt_box, tf.float32) + + height = box[:, 2] - box[:, 0] + width = box[:, 3] - box[:, 1] + center_y = box[:, 0] + 0.5 * height + center_x = box[:, 1] + 0.5 * width + + gt_height = gt_box[:, 2] - gt_box[:, 0] + gt_width = gt_box[:, 3] - gt_box[:, 1] + gt_center_y = gt_box[:, 0] + 0.5 * gt_height + gt_center_x = gt_box[:, 1] + 0.5 * gt_width + + dy = (gt_center_y - center_y) / height + dx = (gt_center_x - center_x) / width + dh = tf.log(gt_height / height) + dw = tf.log(gt_width / width) + + result = tf.stack([dy, dx, dh, dw], axis=1) + return result + + +def box_refinement(box, gt_box): + """Compute refinement needed to transform box to gt_box. + box and gt_box are [N, (y1, x1, y2, x2)]. (y2, x2) is + assumed to be outside the box. + """ + box = box.astype(np.float32) + gt_box = gt_box.astype(np.float32) + + height = box[:, 2] - box[:, 0] + width = box[:, 3] - box[:, 1] + center_y = box[:, 0] + 0.5 * height + center_x = box[:, 1] + 0.5 * width + + gt_height = gt_box[:, 2] - gt_box[:, 0] + gt_width = gt_box[:, 3] - gt_box[:, 1] + gt_center_y = gt_box[:, 0] + 0.5 * gt_height + gt_center_x = gt_box[:, 1] + 0.5 * gt_width + + dy = (gt_center_y - center_y) / height + dx = (gt_center_x - center_x) / width + dh = np.log(gt_height / height) + dw = np.log(gt_width / width) + + return np.stack([dy, dx, dh, dw], axis=1) + + +############################################################ +# Dataset +############################################################ + +class Dataset(object): + """The base class for dataset classes. + To use it, create a new class that adds functions specific to the dataset + you want to use. For example: + + class CatsAndDogsDataset(Dataset): + def load_cats_and_dogs(self): + ... + def load_mask(self, image_id): + ... + def image_reference(self, image_id): + ... + + See COCODataset and ShapesDataset as examples. + """ + + def __init__(self, class_map=None): + self._image_ids = [] + self.image_info = [] + # Background is always the first class + self.class_info = [{"source": "", "id": 0, "name": "BG"}] + self.source_class_ids = {} + + def add_class(self, source, class_id, class_name): + assert "." not in source, "Source name cannot contain a dot" + # Does the class exist already? + for info in self.class_info: + if info['source'] == source and info["id"] == class_id: + # source.class_id combination already available, skip + return + # Add the class + self.class_info.append({ + "source": source, + "id": class_id, + "name": class_name, + }) + + def add_image(self, source, image_id, path, **kwargs): + image_info = { + "id": image_id, + "source": source, + "path": path, + } + image_info.update(kwargs) + self.image_info.append(image_info) + + def image_reference(self, image_id): + """Return a link to the image in its source Website or details about + the image that help looking it up or debugging it. + + Override for your dataset, but pass to this function + if you encounter images not in your dataset. + """ + return "" + + def prepare(self, class_map=None): + """Prepares the Dataset class for use. + + TODO: class map is not supported yet. When done, it should handle mapping + classes from different datasets to the same class ID. + """ + + def clean_name(name): + """Returns a shorter version of object names for cleaner display.""" + return ",".join(name.split(",")[:1]) + + # Build (or rebuild) everything else from the info dicts. + self.num_classes = len(self.class_info) + self.class_ids = np.arange(self.num_classes) + self.class_names = [clean_name(c["name"]) for c in self.class_info] + self.num_images = len(self.image_info) + self._image_ids = np.arange(self.num_images) + + # Mapping from source class and image IDs to internal IDs + self.class_from_source_map = {"{}.{}".format(info['source'], info['id']): id + for info, id in zip(self.class_info, self.class_ids)} + self.image_from_source_map = {"{}.{}".format(info['source'], info['id']): id + for info, id in zip(self.image_info, self.image_ids)} + + # Map sources to class_ids they support + self.sources = list(set([i['source'] for i in self.class_info])) + self.source_class_ids = {} + # Loop over datasets + for source in self.sources: + self.source_class_ids[source] = [] + # Find classes that belong to this dataset + for i, info in enumerate(self.class_info): + # Include BG class in all datasets + if i == 0 or source == info['source']: + self.source_class_ids[source].append(i) + + def map_source_class_id(self, source_class_id): + """Takes a source class ID and returns the int class ID assigned to it. + + For example: + dataset.map_source_class_id("coco.12") -> 23 + """ + return self.class_from_source_map[source_class_id] + + def get_source_class_id(self, class_id, source): + """Map an internal class ID to the corresponding class ID in the source dataset.""" + info = self.class_info[class_id] + assert info['source'] == source + return info['id'] + + def append_data(self, class_info, image_info): + self.external_to_class_id = {} + for i, c in enumerate(self.class_info): + for ds, id in c["map"]: + self.external_to_class_id[ds + str(id)] = i + + # Map external image IDs to internal ones. + self.external_to_image_id = {} + for i, info in enumerate(self.image_info): + self.external_to_image_id[info["ds"] + str(info["id"])] = i + + @property + def image_ids(self): + return self._image_ids + + def source_image_link(self, image_id): + """Returns the path or URL to the image. + Override this to return a URL to the image if it's available online for easy + debugging. + """ + return self.image_info[image_id]["path"] + + def load_image(self, image_id): + """Load the specified image and return a [H,W,3] Numpy array. + """ + # Load image + image = cv2.imread(self.image_info[image_id]['path']) + # image = skimage.io.imread(self.image_info[image_id]['path']) + # If grayscale. Convert to RGB for consistency. + if image.ndim != 3: + image = skimage.color.gray2rgb(image) + # If has an alpha channel, remove it for consistency + if image.shape[-1] == 4: + image = image[..., :3] + return image + + def load_mask(self, image_id): + """Load instance masks for the given image. + + Different datasets use different ways to store masks. Override this + method to load instance masks and return them in the form of am + array of binary masks of shape [height, width, instances]. + + Returns: + masks: A bool array of shape [height, width, instance count] with + a binary mask per instance. + class_ids: a 1D array of class IDs of the instance masks. + """ + # Override this function to load a mask from your dataset. + # Otherwise, it returns an empty mask. + mask = np.empty([0, 0, 0]) + class_ids = np.empty([0], np.int32) + return mask, class_ids + + +def resize_image(image, min_dim=None, max_dim=None, min_scale=None, mode="square"): + """Resizes an image keeping the aspect ratio unchanged. + + min_dim: if provided, resizes the image such that it's smaller + dimension == min_dim + max_dim: if provided, ensures that the image longest side doesn't + exceed this value. + min_scale: if provided, ensure that the image is scaled up by at least + this percent even if min_dim doesn't require it. + mode: Resizing mode. + none: No resizing. Return the image unchanged. + square: Resize and pad with zeros to get a square image + of size [max_dim, max_dim]. + pad64: Pads width and height with zeros to make them multiples of 64. + If min_dim or min_scale are provided, it scales the image up + before padding. max_dim is ignored in this mode. + The multiple of 64 is needed to ensure smooth scaling of feature + maps up and down the 6 levels of the FPN pyramid (2**6=64). + crop: Picks random crops from the image. First, scales the image based + on min_dim and min_scale, then picks a random crop of + size min_dim x min_dim. Can be used in training only. + max_dim is not used in this mode. + + Returns: + image: the resized image + window: (y1, x1, y2, x2). If max_dim is provided, padding might + be inserted in the returned image. If so, this window is the + coordinates of the image part of the full image (excluding + the padding). The x2, y2 pixels are not included. + scale: The scale factor used to resize the image + padding: Padding added to the image [(top, bottom), (left, right), (0, 0)] + """ + # Keep track of image dtype and return results in the same dtype + image_dtype = image.dtype + # Default window (y1, x1, y2, x2) and default scale == 1. + h, w = image.shape[:2] + window = (0, 0, h, w) + scale = 1 + padding = [(0, 0), (0, 0), (0, 0)] + crop = None + + if mode == "none": + return image, window, scale, padding, crop + + # Scale? + if min_dim: + # Scale up but not down + scale = max(1, min_dim / min(h, w)) + if min_scale and scale < min_scale: + scale = min_scale + + # Does it exceed max dim? + if max_dim and mode == "square": + image_max = max(h, w) + if round(image_max * scale) > max_dim: + scale = max_dim / image_max + + # Resize image using bilinear interpolation + if scale != 1: + # image = skimage.transform.resize( + # image, (round(h * scale), round(w * scale)), + # order=1, mode="constant", preserve_range=True) + image = cv2.resize(image, (round(w * scale), round(h * scale))) + + # Need padding or cropping? + if mode == "square": + # Get new height and width + h, w = image.shape[:2] + top_pad = (max_dim - h) // 2 + bottom_pad = max_dim - h - top_pad + left_pad = (max_dim - w) // 2 + right_pad = max_dim - w - left_pad + padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)] + image = np.pad(image, padding, mode='constant', constant_values=0) + window = (top_pad, left_pad, h + top_pad, w + left_pad) + elif mode == "pad64": + h, w = image.shape[:2] + # Both sides must be divisible by 64 + assert min_dim % 64 == 0, "Minimum dimension must be a multiple of 64" + # Height + if h % 64 > 0: + max_h = h - (h % 64) + 64 + top_pad = (max_h - h) // 2 + bottom_pad = max_h - h - top_pad + else: + top_pad = bottom_pad = 0 + # Width + if w % 64 > 0: + max_w = w - (w % 64) + 64 + left_pad = (max_w - w) // 2 + right_pad = max_w - w - left_pad + else: + left_pad = right_pad = 0 + padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)] + image = np.pad(image, padding, mode='constant', constant_values=0) + window = (top_pad, left_pad, h + top_pad, w + left_pad) + elif mode == "crop": + # Pick a random crop + h, w = image.shape[:2] + y = random.randint(0, (h - min_dim)) + x = random.randint(0, (w - min_dim)) + crop = (y, x, min_dim, min_dim) + image = image[y:y + min_dim, x:x + min_dim] + window = (0, 0, min_dim, min_dim) + else: + raise Exception("Mode {} not supported".format(mode)) + return image.astype(image_dtype), window, scale, padding, crop + + +def resize_mask(mask, scale, padding, crop=None): + """Resizes a mask using the given scale and padding. + Typically, you get the scale and padding from resize_image() to + ensure both, the image and the mask, are resized consistently. + + scale: mask scaling factor + padding: Padding to add to the mask in the form + [(top, bottom), (left, right), (0, 0)] + """ + # Suppress warning from scipy 0.13.0, the output shape of zoom() is + # calculated with round() instead of int() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mask = scipy.ndimage.zoom(mask, zoom=[scale, scale, 1], order=0) + if crop is not None: + y, x, h, w = crop + mask = mask[y:y + h, x:x + w] + else: + mask = np.pad(mask, padding, mode='constant', constant_values=0) + return mask + + +def minimize_mask(bbox, mask, mini_shape): + """Resize masks to a smaller version to reduce memory load. + Mini-masks can be resized back to image scale using expand_masks() + + See inspect_data.ipynb notebook for more details. + """ + mini_mask = np.zeros(mini_shape + (mask.shape[-1],), dtype=bool) + for i in range(mask.shape[-1]): + # Pick slice and cast to bool in case load_mask() returned wrong dtype + m = mask[:, :, i].astype(bool) + y1, x1, y2, x2 = bbox[i][:4] + m = m[y1:y2, x1:x2] + if m.size == 0: + raise Exception("Invalid bounding box with area of zero") + # Resize with bilinear interpolation + m = skimage.transform.resize(m, mini_shape, order=1, mode="constant") + mini_mask[:, :, i] = np.around(m).astype(np.bool) + return mini_mask + + +def expand_mask(bbox, mini_mask, image_shape): + """Resizes mini masks back to image size. Reverses the change + of minimize_mask(). + + See inspect_data.ipynb notebook for more details. + """ + mask = np.zeros(image_shape[:2] + (mini_mask.shape[-1],), dtype=bool) + for i in range(mask.shape[-1]): + m = mini_mask[:, :, i] + y1, x1, y2, x2 = bbox[i][:4] + h = y2 - y1 + w = x2 - x1 + # Resize with bilinear interpolation + m = skimage.transform.resize(m, (h, w), order=1, mode="constant") + mask[y1:y2, x1:x2, i] = np.around(m).astype(np.bool) + return mask + + +def unmold_mask(mask, bbox, image_shape): + """Converts a mask generated by the neural network to a format similar + to its original shape. + mask: [height, width] of type float. A small, typically 28x28 mask. + bbox: [y1, x1, y2, x2]. The box to fit the mask in. + + Returns a binary mask with the same size as the original image. + """ + threshold = 0.5 + y1, x1, y2, x2 = bbox + # mask = skimage.transform.resize(mask, (y2 - y1, x2 - x1), order=1, mode="constant") + mask = cv2.resize(mask, (x2 - x1, y2 - y1)) + mask = np.where(mask >= threshold, 1, 0).astype(np.bool) + + # Put the mask in the right location. + full_mask = np.zeros(image_shape[:2], dtype=np.bool) + full_mask[y1:y2, x1:x2] = mask + return full_mask + + +############################################################ +# Anchors +############################################################ + +def generate_anchors(scales, ratios, shape, feature_stride, anchor_stride): + """ + scales: 1D array of anchor sizes in pixels. Example: [32, 64, 128] + ratios: 1D array of anchor ratios of width/height. Example: [0.5, 1, 2] + shape: [height, width] spatial shape of the feature map over which + to generate anchors. + feature_stride: Stride of the feature map relative to the image in pixels. + anchor_stride: Stride of anchors on the feature map. For example, if the + value is 2 then generate anchors for every other feature map pixel. + """ + # Get all combinations of scales and ratios + scales, ratios = np.meshgrid(np.array(scales), np.array(ratios)) + scales = scales.flatten() + ratios = ratios.flatten() + + # Enumerate heights and widths from scales and ratios + heights = scales / np.sqrt(ratios) + widths = scales * np.sqrt(ratios) + + # Enumerate shifts in feature space + shifts_y = np.arange(0, shape[0], anchor_stride) * feature_stride + shifts_x = np.arange(0, shape[1], anchor_stride) * feature_stride + shifts_x, shifts_y = np.meshgrid(shifts_x, shifts_y) + + # Enumerate combinations of shifts, widths, and heights + box_widths, box_centers_x = np.meshgrid(widths, shifts_x) + box_heights, box_centers_y = np.meshgrid(heights, shifts_y) + + # Reshape to get a list of (y, x) and a list of (h, w) + box_centers = np.stack( + [box_centers_y, box_centers_x], axis=2).reshape([-1, 2]) + box_sizes = np.stack([box_heights, box_widths], axis=2).reshape([-1, 2]) + + # Convert to corner coordinates (y1, x1, y2, x2) + boxes = np.concatenate([box_centers - 0.5 * box_sizes, + box_centers + 0.5 * box_sizes], axis=1) + return boxes + + +def generate_pyramid_anchors(scales, ratios, feature_shapes, feature_strides, + anchor_stride): + """Generate anchors at different levels of a feature pyramid. Each scale + is associated with a level of the pyramid, but each ratio is used in + all levels of the pyramid. + + Returns: + anchors: [N, (y1, x1, y2, x2)]. All generated anchors in one array. Sorted + with the same order of the given scales. So, anchors of scale[0] come + first, then anchors of scale[1], and so on. + """ + # Anchors + # [anchor_count, (y1, x1, y2, x2)] + anchors = [] + for i in range(len(scales)): + anchors.append(generate_anchors(scales[i], ratios, feature_shapes[i], + feature_strides[i], anchor_stride)) + return np.concatenate(anchors, axis=0) + + +############################################################ +# Miscellaneous +############################################################ + +def trim_zeros(x): + """It's common to have tensors larger than the available data and + pad with zeros. This function removes rows that are all zeros. + + x: [rows, columns]. + """ + assert len(x.shape) == 2 + return x[~np.all(x == 0, axis=1)] + + +def compute_matches(gt_boxes, gt_class_ids, gt_masks, + pred_boxes, pred_class_ids, pred_scores, pred_masks, + iou_threshold=0.5, score_threshold=0.0): + """Finds matches between prediction and ground truth instances. + + Returns: + gt_match: 1-D array. For each GT box it has the index of the matched + predicted box. + pred_match: 1-D array. For each predicted box, it has the index of + the matched ground truth box. + overlaps: [pred_boxes, gt_boxes] IoU overlaps. + """ + # Trim zero padding + # TODO: cleaner to do zero unpadding upstream + gt_boxes = trim_zeros(gt_boxes) + gt_masks = gt_masks[..., :gt_boxes.shape[0]] + pred_boxes = trim_zeros(pred_boxes) + pred_scores = pred_scores[:pred_boxes.shape[0]] + # Sort predictions by score from high to low + indices = np.argsort(pred_scores)[::-1] + pred_boxes = pred_boxes[indices] + pred_class_ids = pred_class_ids[indices] + pred_scores = pred_scores[indices] + pred_masks = pred_masks[..., indices] + + # Compute IoU overlaps [pred_masks, gt_masks] + overlaps = compute_overlaps_masks(pred_masks, gt_masks) + + # Loop through predictions and find matching ground truth boxes + match_count = 0 + pred_match = -1 * np.ones([pred_boxes.shape[0]]) + gt_match = -1 * np.ones([gt_boxes.shape[0]]) + for i in range(len(pred_boxes)): + # Find best matching ground truth box + # 1. Sort matches by score + sorted_ixs = np.argsort(overlaps[i])[::-1] + # 2. Remove low scores + low_score_idx = np.where(overlaps[i, sorted_ixs] < score_threshold)[0] + if low_score_idx.size > 0: + sorted_ixs = sorted_ixs[:low_score_idx[0]] + # 3. Find the match + for j in sorted_ixs: + # If ground truth box is already matched, go to next one + if gt_match[j] > 0: + continue + # If we reach IoU smaller than the threshold, end the loop + iou = overlaps[i, j] + if iou < iou_threshold: + break + # Do we have a match? + if pred_class_ids[i] == gt_class_ids[j]: + match_count += 1 + gt_match[j] = i + pred_match[i] = j + break + + return gt_match, pred_match, overlaps + + +def compute_ap(gt_boxes, gt_class_ids, gt_masks, + pred_boxes, pred_class_ids, pred_scores, pred_masks, + iou_threshold=0.5): + """Compute Average Precision at a set IoU threshold (default 0.5). + + Returns: + mAP: Mean Average Precision + precisions: List of precisions at different class score thresholds. + recalls: List of recall values at different class score thresholds. + overlaps: [pred_boxes, gt_boxes] IoU overlaps. + """ + # Get matches and overlaps + gt_match, pred_match, overlaps = compute_matches( + gt_boxes, gt_class_ids, gt_masks, + pred_boxes, pred_class_ids, pred_scores, pred_masks, + iou_threshold) + + # Compute precision and recall at each prediction box step + precisions = np.cumsum(pred_match > -1) / (np.arange(len(pred_match)) + 1) + recalls = np.cumsum(pred_match > -1).astype(np.float32) / len(gt_match) + + # Pad with start and end values to simplify the math + precisions = np.concatenate([[0], precisions, [0]]) + recalls = np.concatenate([[0], recalls, [1]]) + + # Ensure precision values decrease but don't increase. This way, the + # precision value at each recall threshold is the maximum it can be + # for all following recall thresholds, as specified by the VOC paper. + for i in range(len(precisions) - 2, -1, -1): + precisions[i] = np.maximum(precisions[i], precisions[i + 1]) + + # Compute mean AP over recall range + indices = np.where(recalls[:-1] != recalls[1:])[0] + 1 + mAP = np.sum((recalls[indices] - recalls[indices - 1]) * + precisions[indices]) + + return mAP, precisions, recalls, overlaps + + +def compute_ap_range(gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + iou_thresholds=None, verbose=1): + """Compute AP over a range or IoU thresholds. Default range is 0.5-0.95.""" + # Default is 0.5 to 0.95 with increments of 0.05 + iou_thresholds = iou_thresholds or np.arange(0.5, 1.0, 0.05) + + # Compute AP over range of IoU thresholds + AP = [] + for iou_threshold in iou_thresholds: + ap, precisions, recalls, overlaps = \ + compute_ap(gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + iou_threshold=iou_threshold) + if verbose: + print("AP @{:.2f}:\t {:.3f}".format(iou_threshold, ap)) + AP.append(ap) + AP = np.array(AP).mean() + if verbose: + print("AP @{:.2f}-{:.2f}:\t {:.3f}".format( + iou_thresholds[0], iou_thresholds[-1], AP)) + return AP + + +def compute_recall(pred_boxes, gt_boxes, iou): + """Compute the recall at the given IoU threshold. It's an indication + of how many GT boxes were found by the given prediction boxes. + + pred_boxes: [N, (y1, x1, y2, x2)] in image coordinates + gt_boxes: [N, (y1, x1, y2, x2)] in image coordinates + """ + # Measure overlaps + overlaps = compute_overlaps(pred_boxes, gt_boxes) + iou_max = np.max(overlaps, axis=1) + iou_argmax = np.argmax(overlaps, axis=1) + positive_ids = np.where(iou_max >= iou)[0] + matched_gt_boxes = iou_argmax[positive_ids] + + recall = len(set(matched_gt_boxes)) / gt_boxes.shape[0] + return recall, positive_ids + + +# ## Batch Slicing +# Some custom layers support a batch size of 1 only, and require a lot of work +# to support batches greater than 1. This function slices an input tensor +# across the batch dimension and feeds batches of size 1. Effectively, +# an easy way to support batches > 1 quickly with little code modification. +# In the long run, it's more efficient to modify the code to support large +# batches and getting rid of this function. Consider this a temporary solution +def batch_slice(inputs, graph_fn, batch_size, names=None): + """Splits inputs into slices and feeds each slice to a copy of the given + computation graph and then combines the results. It allows you to run a + graph on a batch of inputs even if the graph is written to support one + instance only. + + inputs: list of tensors. All must have the same first dimension length + graph_fn: A function that returns a TF tensor that's part of a graph. + batch_size: number of slices to divide the data into. + names: If provided, assigns names to the resulting tensors. + """ + if not isinstance(inputs, list): + inputs = [inputs] + + outputs = [] + for i in range(batch_size): + inputs_slice = [x[i] for x in inputs] + output_slice = graph_fn(*inputs_slice) + if not isinstance(output_slice, (tuple, list)): + output_slice = [output_slice] + outputs.append(output_slice) + # Change outputs from a list of slices where each is + # a list of outputs to a list of outputs and each has + # a list of slices + outputs = list(zip(*outputs)) + + if names is None: + names = [None] * len(outputs) + + result = [tf.stack(o, axis=0, name=n) + for o, n in zip(outputs, names)] + if len(result) == 1: + result = result[0] + + return result + + +def download_trained_weights(coco_model_path, verbose=1): + """Download COCO trained weights from Releases. + + coco_model_path: local path of COCO trained weights + """ + if verbose > 0: + print("Downloading pretrained model to " + coco_model_path + " ...") + with urllib.request.urlopen(COCO_MODEL_URL) as resp, open(coco_model_path, 'wb') as out: + shutil.copyfileobj(resp, out) + if verbose > 0: + print("... done downloading pretrained model!") + + +def norm_boxes(boxes, shape): + """Converts boxes from pixel coordinates to normalized coordinates. + boxes: [N, (y1, x1, y2, x2)] in pixel coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [N, (y1, x1, y2, x2)] in normalized coordinates + """ + h, w = shape + scale = np.array([h - 1, w - 1, h - 1, w - 1]) + shift = np.array([0, 0, 1, 1]) + return np.divide((boxes - shift), scale).astype(np.float32) + + +def denorm_boxes(boxes, shape): + """Converts boxes from normalized coordinates to pixel coordinates. + boxes: [N, (y1, x1, y2, x2)] in normalized coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [N, (y1, x1, y2, x2)] in pixel coordinates + """ + h, w = shape + scale = np.array([h - 1, w - 1, h - 1, w - 1]) + shift = np.array([0, 0, 1, 1]) + return np.around(np.multiply(boxes, scale) + shift).astype(np.int32) diff --git a/mask_rcnn/tf_servable/inferencing/visualize.py b/mask_rcnn/tf_servable/inferencing/visualize.py new file mode 100644 index 00000000..80e5ef58 --- /dev/null +++ b/mask_rcnn/tf_servable/inferencing/visualize.py @@ -0,0 +1,502 @@ +""" +Mask R-CNN +Display and Visualization Functions. + +Copyright (c) 2017 Matterport, Inc. +Licensed under the MIT License (see LICENSE for details) +Written by Waleed Abdulla +""" +import warnings +warnings.filterwarnings('ignore', category=DeprecationWarning) +warnings.filterwarnings('ignore', category=FutureWarning) +import os +import sys +import random +import itertools +import colorsys + +import numpy as np +from skimage.measure import find_contours +import matplotlib.pyplot as plt +from matplotlib import patches, lines +from matplotlib.patches import Polygon +import IPython.display + +# Root directory of the project +ROOT_DIR = os.path.abspath("../") + +# Import Mask RCNN +sys.path.append(ROOT_DIR) # To find local version of the library +from mrcnn import utils + + +############################################################ +# Visualization +############################################################ + +def display_images(images, titles=None, cols=4, cmap=None, norm=None, + interpolation=None): + """Display the given set of images, optionally with titles. + images: list or array of image tensors in HWC format. + titles: optional. A list of titles to display with each image. + cols: number of images per row + cmap: Optional. Color map to use. For example, "Blues". + norm: Optional. A Normalize instance to map values to colors. + interpolation: Optional. Image interpolation to use for display. + """ + titles = titles if titles is not None else [""] * len(images) + rows = len(images) // cols + 1 + plt.figure(figsize=(14, 14 * rows // cols)) + i = 1 + for image, title in zip(images, titles): + plt.subplot(rows, cols, i) + plt.title(title, fontsize=9) + plt.axis('off') + plt.imshow(image.astype(np.uint8), cmap=cmap, + norm=norm, interpolation=interpolation) + i += 1 + plt.show() + + +def random_colors(N, bright=True): + """ + Generate random colors. + To get visually distinct colors, generate them in HSV space then + convert to RGB. + """ + brightness = 1.0 if bright else 0.7 + hsv = [(i / N, 1, brightness) for i in range(N)] + colors = list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv)) + random.shuffle(colors) + return colors + + +def apply_mask(image, mask, color, alpha=0.5): + """Apply the given mask to the image. + """ + for c in range(3): + image[:, :, c] = np.where(mask == 1, + image[:, :, c] * + (1 - alpha) + alpha * color[c] * 255, + image[:, :, c]) + return image + + +def display_instances(image, boxes, masks, class_ids, class_names, + scores=None, title="", + figsize=(16, 16), ax=None, + show_mask=True, show_bbox=True, + colors=None, captions=None): + """ + boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates. + masks: [height, width, num_instances] + class_ids: [num_instances] + class_names: list of class names of the dataset + scores: (optional) confidence scores for each box + title: (optional) Figure title + show_mask, show_bbox: To show masks and bounding boxes or not + figsize: (optional) the size of the image + colors: (optional) An array or colors to use with each object + captions: (optional) A list of strings to use as captions for each object + """ + # Number of instances + N = boxes.shape[0] + if not N: + print("\n*** No instances to display *** \n") + else: + assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0] + + # If no axis is passed, create one and automatically call show() + auto_show = False + if not ax: + _, ax = plt.subplots(1, figsize=figsize) + auto_show = True + + # Generate random colors + colors = colors or random_colors(N) + + # Show area outside image boundaries. + height, width = image.shape[:2] + ax.set_ylim(height + 10, -10) + ax.set_xlim(-10, width + 10) + ax.axis('off') + ax.set_title(title) + + masked_image = image.astype(np.uint32).copy() + for i in range(N): + color = colors[i] + + # Bounding box + if not np.any(boxes[i]): + # Skip this instance. Has no bbox. Likely lost in image cropping. + continue + y1, x1, y2, x2 = boxes[i] + if show_bbox: + p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, + alpha=0.7, linestyle="dashed", + edgecolor=color, facecolor='none') + ax.add_patch(p) + + # Label + if not captions: + class_id = class_ids[i] + score = scores[i] if scores is not None else None + label = class_names[class_id] + caption = "{} {:.3f}".format(label, score) if score else label + else: + caption = captions[i] + ax.text(x1, y1 + 8, caption, + color='w', size=11, backgroundcolor="none") + + # Mask + mask = masks[:, :, i] + if show_mask: + masked_image = apply_mask(masked_image, mask, color) + + # Mask Polygon + # Pad to ensure proper polygons for masks that touch image edges. + padded_mask = np.zeros( + (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8) + padded_mask[1:-1, 1:-1] = mask + contours = find_contours(padded_mask, 0.5) + for verts in contours: + # Subtract the padding and flip (y, x) to (x, y) + verts = np.fliplr(verts) - 1 + p = Polygon(verts, facecolor="none", edgecolor=color) + ax.add_patch(p) + ax.imshow(masked_image.astype(np.uint8)) + if auto_show: + plt.show() + + +def display_differences(image, + gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + class_names, title="", ax=None, + show_mask=True, show_box=True, + iou_threshold=0.5, score_threshold=0.5): + """Display ground truth and prediction instances on the same image.""" + # Match predictions to ground truth + gt_match, pred_match, overlaps = utils.compute_matches( + gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + iou_threshold=iou_threshold, score_threshold=score_threshold) + # Ground truth = green. Predictions = red + colors = [(0, 1, 0, .8)] * len(gt_match)\ + + [(1, 0, 0, 1)] * len(pred_match) + # Concatenate GT and predictions + class_ids = np.concatenate([gt_class_id, pred_class_id]) + scores = np.concatenate([np.zeros([len(gt_match)]), pred_score]) + boxes = np.concatenate([gt_box, pred_box]) + masks = np.concatenate([gt_mask, pred_mask], axis=-1) + # Captions per instance show score/IoU + captions = ["" for m in gt_match] + ["{:.2f} / {:.2f}".format( + pred_score[i], + (overlaps[i, int(pred_match[i])] + if pred_match[i] > -1 else overlaps[i].max())) + for i in range(len(pred_match))] + # Set title if not provided + title = title or "Ground Truth and Detections\n GT=green, pred=red, captions: score/IoU" + # Display + display_instances( + image, + boxes, masks, class_ids, + class_names, scores, ax=ax, + show_bbox=show_box, show_mask=show_mask, + colors=colors, captions=captions, + title=title) + + +def draw_rois(image, rois, refined_rois, mask, class_ids, class_names, limit=10): + """ + anchors: [n, (y1, x1, y2, x2)] list of anchors in image coordinates. + proposals: [n, 4] the same anchors but refined to fit objects better. + """ + masked_image = image.copy() + + # Pick random anchors in case there are too many. + ids = np.arange(rois.shape[0], dtype=np.int32) + ids = np.random.choice( + ids, limit, replace=False) if ids.shape[0] > limit else ids + + fig, ax = plt.subplots(1, figsize=(12, 12)) + if rois.shape[0] > limit: + plt.title("Showing {} random ROIs out of {}".format( + len(ids), rois.shape[0])) + else: + plt.title("{} ROIs".format(len(ids))) + + # Show area outside image boundaries. + ax.set_ylim(image.shape[0] + 20, -20) + ax.set_xlim(-50, image.shape[1] + 20) + ax.axis('off') + + for i, id in enumerate(ids): + color = np.random.rand(3) + class_id = class_ids[id] + # ROI + y1, x1, y2, x2 = rois[id] + p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, + edgecolor=color if class_id else "gray", + facecolor='none', linestyle="dashed") + ax.add_patch(p) + # Refined ROI + if class_id: + ry1, rx1, ry2, rx2 = refined_rois[id] + p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2, + edgecolor=color, facecolor='none') + ax.add_patch(p) + # Connect the top-left corners of the anchor and proposal for easy visualization + ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color)) + + # Label + label = class_names[class_id] + ax.text(rx1, ry1 + 8, "{}".format(label), + color='w', size=11, backgroundcolor="none") + + # Mask + m = utils.unmold_mask(mask[id], rois[id] + [:4].astype(np.int32), image.shape) + masked_image = apply_mask(masked_image, m, color) + + ax.imshow(masked_image) + + # Print stats + print("Positive ROIs: ", class_ids[class_ids > 0].shape[0]) + print("Negative ROIs: ", class_ids[class_ids == 0].shape[0]) + print("Positive Ratio: {:.2f}".format( + class_ids[class_ids > 0].shape[0] / class_ids.shape[0])) + + +# TODO: Replace with matplotlib equivalent? +def draw_box(image, box, color): + """Draw 3-pixel width bounding boxes on the given image array. + color: list of 3 int values for RGB. + """ + y1, x1, y2, x2 = box + image[y1:y1 + 2, x1:x2] = color + image[y2:y2 + 2, x1:x2] = color + image[y1:y2, x1:x1 + 2] = color + image[y1:y2, x2:x2 + 2] = color + return image + + +def display_top_masks(image, mask, class_ids, class_names, limit=4): + """Display the given image and the top few class masks.""" + to_display = [] + titles = [] + to_display.append(image) + titles.append("H x W={}x{}".format(image.shape[0], image.shape[1])) + # Pick top prominent classes in this image + unique_class_ids = np.unique(class_ids) + mask_area = [np.sum(mask[:, :, np.where(class_ids == i)[0]]) + for i in unique_class_ids] + top_ids = [v[0] for v in sorted(zip(unique_class_ids, mask_area), + key=lambda r: r[1], reverse=True) if v[1] > 0] + # Generate images and titles + for i in range(limit): + class_id = top_ids[i] if i < len(top_ids) else -1 + # Pull masks of instances belonging to the same class. + m = mask[:, :, np.where(class_ids == class_id)[0]] + m = np.sum(m * np.arange(1, m.shape[-1] + 1), -1) + to_display.append(m) + titles.append(class_names[class_id] if class_id != -1 else "-") + display_images(to_display, titles=titles, cols=limit + 1, cmap="Blues_r") + + +def plot_precision_recall(AP, precisions, recalls): + """Draw the precision-recall curve. + + AP: Average precision at IoU >= 0.5 + precisions: list of precision values + recalls: list of recall values + """ + # Plot the Precision-Recall curve + _, ax = plt.subplots(1) + ax.set_title("Precision-Recall Curve. AP@50 = {:.3f}".format(AP)) + ax.set_ylim(0, 1.1) + ax.set_xlim(0, 1.1) + _ = ax.plot(recalls, precisions) + + +def plot_overlaps(gt_class_ids, pred_class_ids, pred_scores, + overlaps, class_names, threshold=0.5): + """Draw a grid showing how ground truth objects are classified. + gt_class_ids: [N] int. Ground truth class IDs + pred_class_id: [N] int. Predicted class IDs + pred_scores: [N] float. The probability scores of predicted classes + overlaps: [pred_boxes, gt_boxes] IoU overlaps of predictions and GT boxes. + class_names: list of all class names in the dataset + threshold: Float. The prediction probability required to predict a class + """ + gt_class_ids = gt_class_ids[gt_class_ids != 0] + pred_class_ids = pred_class_ids[pred_class_ids != 0] + + plt.figure(figsize=(12, 10)) + plt.imshow(overlaps, interpolation='nearest', cmap=plt.cm.Blues) + plt.yticks(np.arange(len(pred_class_ids)), + ["{} ({:.2f})".format(class_names[int(id)], pred_scores[i]) + for i, id in enumerate(pred_class_ids)]) + plt.xticks(np.arange(len(gt_class_ids)), + [class_names[int(id)] for id in gt_class_ids], rotation=90) + + thresh = overlaps.max() / 2. + for i, j in itertools.product(range(overlaps.shape[0]), + range(overlaps.shape[1])): + text = "" + if overlaps[i, j] > threshold: + text = "match" if gt_class_ids[j] == pred_class_ids[i] else "wrong" + color = ("white" if overlaps[i, j] > thresh + else "black" if overlaps[i, j] > 0 + else "grey") + plt.text(j, i, "{:.3f}\n{}".format(overlaps[i, j], text), + horizontalalignment="center", verticalalignment="center", + fontsize=9, color=color) + + plt.tight_layout() + plt.xlabel("Ground Truth") + plt.ylabel("Predictions") + + +def draw_boxes(image, boxes=None, refined_boxes=None, + masks=None, captions=None, visibilities=None, + title="", ax=None): + """Draw bounding boxes and segmentation masks with different + customizations. + + boxes: [N, (y1, x1, y2, x2, class_id)] in image coordinates. + refined_boxes: Like boxes, but draw with solid lines to show + that they're the result of refining 'boxes'. + masks: [N, height, width] + captions: List of N titles to display on each box + visibilities: (optional) List of values of 0, 1, or 2. Determine how + prominent each bounding box should be. + title: An optional title to show over the image + ax: (optional) Matplotlib axis to draw on. + """ + # Number of boxes + assert boxes is not None or refined_boxes is not None + N = boxes.shape[0] if boxes is not None else refined_boxes.shape[0] + + # Matplotlib Axis + if not ax: + _, ax = plt.subplots(1, figsize=(12, 12)) + + # Generate random colors + colors = random_colors(N) + + # Show area outside image boundaries. + margin = image.shape[0] // 10 + ax.set_ylim(image.shape[0] + margin, -margin) + ax.set_xlim(-margin, image.shape[1] + margin) + ax.axis('off') + + ax.set_title(title) + + masked_image = image.astype(np.uint32).copy() + for i in range(N): + # Box visibility + visibility = visibilities[i] if visibilities is not None else 1 + if visibility == 0: + color = "gray" + style = "dotted" + alpha = 0.5 + elif visibility == 1: + color = colors[i] + style = "dotted" + alpha = 1 + elif visibility == 2: + color = colors[i] + style = "solid" + alpha = 1 + + # Boxes + if boxes is not None: + if not np.any(boxes[i]): + # Skip this instance. Has no bbox. Likely lost in cropping. + continue + y1, x1, y2, x2 = boxes[i] + p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, + alpha=alpha, linestyle=style, + edgecolor=color, facecolor='none') + ax.add_patch(p) + + # Refined boxes + if refined_boxes is not None and visibility > 0: + ry1, rx1, ry2, rx2 = refined_boxes[i].astype(np.int32) + p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2, + edgecolor=color, facecolor='none') + ax.add_patch(p) + # Connect the top-left corners of the anchor and proposal + if boxes is not None: + ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color)) + + # Captions + if captions is not None: + caption = captions[i] + # If there are refined boxes, display captions on them + if refined_boxes is not None: + y1, x1, y2, x2 = ry1, rx1, ry2, rx2 + ax.text(x1, y1, caption, size=11, verticalalignment='top', + color='w', backgroundcolor="none", + bbox={'facecolor': color, 'alpha': 0.5, + 'pad': 2, 'edgecolor': 'none'}) + + # Masks + if masks is not None: + mask = masks[:, :, i] + masked_image = apply_mask(masked_image, mask, color) + # Mask Polygon + # Pad to ensure proper polygons for masks that touch image edges. + padded_mask = np.zeros( + (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8) + padded_mask[1:-1, 1:-1] = mask + contours = find_contours(padded_mask, 0.5) + for verts in contours: + # Subtract the padding and flip (y, x) to (x, y) + verts = np.fliplr(verts) - 1 + p = Polygon(verts, facecolor="none", edgecolor=color) + ax.add_patch(p) + ax.imshow(masked_image.astype(np.uint8)) + + +def display_table(table): + """Display values in a table format. + table: an iterable of rows, and each row is an iterable of values. + """ + html = "" + for row in table: + row_html = "" + for col in row: + row_html += "{:40}".format(str(col)) + html += "" + row_html + "" + html = "" + html + "
    " + IPython.display.display(IPython.display.HTML(html)) + + +def display_weight_stats(model): + """Scans all the weights in the model and returns a list of tuples + that contain stats about each weight. + """ + layers = model.get_trainable_layers() + table = [["WEIGHT NAME", "SHAPE", "MIN", "MAX", "STD"]] + for l in layers: + weight_values = l.get_weights() # list of Numpy arrays + weight_tensors = l.weights # list of TF tensors + for i, w in enumerate(weight_values): + weight_name = weight_tensors[i].name + # Detect problematic layers. Exclude biases of conv layers. + alert = "" + if w.min() == w.max() and not (l.__class__.__name__ == "Conv2D" and i == 1): + alert += "*** dead?" + if np.abs(w.min()) > 1000 or np.abs(w.max()) > 1000: + alert += "*** Overflow?" + # Add row + table.append([ + weight_name + alert, + str(w.shape), + "{:+9.4f}".format(w.min()), + "{:+10.4f}".format(w.max()), + "{:+9.4f}".format(w.std()), + ]) + display_table(table) diff --git a/mask_rcnn/tf_servable/main.py b/mask_rcnn/tf_servable/main.py new file mode 100644 index 00000000..25700221 --- /dev/null +++ b/mask_rcnn/tf_servable/main.py @@ -0,0 +1,118 @@ +from user_config import * +import tensorflow as tf +import keras.backend as K +from tensorflow.python.saved_model import signature_constants +from tensorflow.python.saved_model import tag_constants +import os +from config import mask_config +from model import MaskRCNN + +sess = tf.Session() +K.set_session(sess) + + +def get_config(): + if is_coco: + import coco + class InferenceConfig(coco.CocoConfig): + GPU_COUNT = 1 + IMAGES_PER_GPU = 1 + + config = InferenceConfig() + + else: + config = mask_config(NUMBER_OF_CLASSES) + + return config + + +def freeze_session(session, keep_var_names=None, output_names=None, clear_devices=True): + graph = sess.graph + + with graph.as_default(): + freeze_var_names = list(set(v.op.name for v in tf.global_variables()).difference(keep_var_names or [])) + + output_names = output_names or [] + input_graph_def = graph.as_graph_def() + + if clear_devices: + for node in input_graph_def.node: + node.device = "" + + frozen_graph = tf.graph_util.convert_variables_to_constants( + session, input_graph_def, output_names, freeze_var_names) + return frozen_graph + + +def freeze_model(model, name): + frozen_graph = freeze_session( + sess, + output_names=[out.op.name for out in model.outputs][:4]) + directory = PATH_TO_SAVE_FROZEN_PB + tf.train.write_graph(frozen_graph, directory, name , as_text=False) + print("*"*80) + print("Finish converting keras model to Frozen PB") + print('PATH: ', PATH_TO_SAVE_FROZEN_PB) + print("*" * 80) + + +def make_serving_ready(model_path, save_serve_path, version_number): + import tensorflow as tf + + export_dir = os.path.join(save_serve_path, str(version_number)) + graph_pb = model_path + + builder = tf.saved_model.builder.SavedModelBuilder(export_dir) + + with tf.gfile.GFile(graph_pb, "rb") as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + + sigs = {} + + with tf.Session(graph=tf.Graph()) as sess: + # name="" is important to ensure we don't get spurious prefixing + tf.import_graph_def(graph_def, name="") + g = tf.get_default_graph() + input_image = g.get_tensor_by_name("input_image:0") + input_image_meta = g.get_tensor_by_name("input_image_meta:0") + input_anchors = g.get_tensor_by_name("input_anchors:0") + + output_detection = g.get_tensor_by_name("mrcnn_detection/Reshape_1:0") + output_mask = g.get_tensor_by_name("mrcnn_mask/Reshape_1:0") + + sigs[signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] = \ + tf.saved_model.signature_def_utils.predict_signature_def( + {"input_image": input_image, 'input_image_meta': input_image_meta, 'input_anchors': input_anchors}, + {"mrcnn_detection/Reshape_1": output_detection, 'mrcnn_mask/Reshape_1': output_mask}) + + builder.add_meta_graph_and_variables(sess, + [tag_constants.SERVING], + signature_def_map=sigs) + + builder.save() + print("*" * 80) + print("FINISH CONVERTING FROZEN PB TO SERVING READY") + print("PATH:", PATH_TO_SAVE_TENSORFLOW_SERVING_MODEL) + print("*" * 80) + + +# Load Mask RCNN config +# you can also load your own config in here. +# config = your_custom_config_class +config = get_config() + + +# LOAD MODEL +model = MaskRCNN(mode="inference", model_dir=MODEL_DIR, config=config) +model.load_weights(H5_WEIGHT_PATH, by_name=True) + +# Converting keras model to PB frozen graph +freeze_model(model.keras_model, FROZEN_NAME) + +# Now convert frozen graph to Tensorflow Serving Ready +make_serving_ready(os.path.join(PATH_TO_SAVE_FROZEN_PB, FROZEN_NAME), + PATH_TO_SAVE_TENSORFLOW_SERVING_MODEL, + VERSION_NUMBER) + +print("COMPLETED") \ No newline at end of file diff --git a/mask_rcnn/tf_servable/model.py b/mask_rcnn/tf_servable/model.py new file mode 100644 index 00000000..5dcf67e0 --- /dev/null +++ b/mask_rcnn/tf_servable/model.py @@ -0,0 +1,2882 @@ + +import utils +import keras.backend as K +import tensorflow as tf +import os +import random +import datetime +import re +import math +import logging +from collections import OrderedDict +import multiprocessing +import numpy as np +import skimage.transform +import keras +import keras.layers as KL +import keras.engine as KE +import keras.models as KM +import os +from tensorflow.python.saved_model import signature_constants +from tensorflow.python.saved_model import tag_constants + +# Requires TensorFlow 1.3+ and Keras 2.0.8+. +from distutils.version import LooseVersion + +assert LooseVersion(tf.__version__) >= LooseVersion("1.3") +assert LooseVersion(keras.__version__) >= LooseVersion('2.0.8') + + +############################################################ +# Utility Functions +############################################################ + +def log(text, array=None): + """Prints a text message. And, optionally, if a Numpy array is provided it + prints it's shape, min, and max values. + """ + if array is not None: + text = text.ljust(25) + text += ("shape: {:20} min: {:10.5f} max: {:10.5f} {}".format( + str(array.shape), + array.min() if array.size else "", + array.max() if array.size else "", + array.dtype)) + + +class BatchNorm(KL.BatchNormalization): + """Extends the Keras BatchNormalization class to allow a central place + to make changes if needed. + + Batch normalization has a negative effect on training if batches are small + so this layer is often frozen (via setting in Config class) and functions + as linear layer. + """ + + def call(self, inputs, training=None): + """ + Note about training values: + None: Train BN layers. This is the normal mode + False: Freeze BN layers. Good when batch size is small + True: (don't use). Set layer in training mode even when making inferences + """ + return super(self.__class__, self).call(inputs, training=training) + + +def compute_backbone_shapes(config, image_shape): + """Computes the width and height of each stage of the backbone network. + + Returns: + [N, (height, width)]. Where N is the number of stages + """ + if callable(config.BACKBONE): + return config.COMPUTE_BACKBONE_SHAPE(image_shape) + + # Currently supports ResNet only + assert config.BACKBONE in ["resnet50", "resnet101"] + return np.array( + [[int(math.ceil(image_shape[0] / stride)), + int(math.ceil(image_shape[1] / stride))] + for stride in config.BACKBONE_STRIDES]) + + +############################################################ +# Resnet Graph +############################################################ + +# Code adopted from: +# https://github.com/fchollet/deep-learning-models/blob/master/resnet50.py + +def identity_block(input_tensor, kernel_size, filters, stage, block, + use_bias=True, train_bn=True): + """The identity_block is the block that has no conv layer at shortcut + # Arguments + input_tensor: input tensor + kernel_size: default 3, the kernel size of middle conv layer at main path + filters: list of integers, the nb_filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + use_bias: Boolean. To use or not use a bias in conv layers. + train_bn: Boolean. Train or freeze Batch Norm layers + """ + nb_filter1, nb_filter2, nb_filter3 = filters + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = KL.Conv2D(nb_filter1, (1, 1), name=conv_name_base + '2a', + use_bias=use_bias)(input_tensor) + x = BatchNorm(name=bn_name_base + '2a')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.Conv2D(nb_filter2, (kernel_size, kernel_size), padding='same', + name=conv_name_base + '2b', use_bias=use_bias)(x) + x = BatchNorm(name=bn_name_base + '2b')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.Conv2D(nb_filter3, (1, 1), name=conv_name_base + '2c', + use_bias=use_bias)(x) + x = BatchNorm(name=bn_name_base + '2c')(x, training=train_bn) + + x = KL.Add()([x, input_tensor]) + x = KL.Activation('relu', name='res' + str(stage) + block + '_out')(x) + return x + + +def conv_block(input_tensor, kernel_size, filters, stage, block, + strides=(2, 2), use_bias=True, train_bn=True): + """conv_block is the block that has a conv layer at shortcut + # Arguments + input_tensor: input tensor + kernel_size: default 3, the kernel size of middle conv layer at main path + filters: list of integers, the nb_filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + use_bias: Boolean. To use or not use a bias in conv layers. + train_bn: Boolean. Train or freeze Batch Norm layers + Note that from stage 3, the first conv layer at main path is with subsample=(2,2) + And the shortcut should have subsample=(2,2) as well + """ + nb_filter1, nb_filter2, nb_filter3 = filters + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = KL.Conv2D(nb_filter1, (1, 1), strides=strides, + name=conv_name_base + '2a', use_bias=use_bias)(input_tensor) + x = BatchNorm(name=bn_name_base + '2a')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.Conv2D(nb_filter2, (kernel_size, kernel_size), padding='same', + name=conv_name_base + '2b', use_bias=use_bias)(x) + x = BatchNorm(name=bn_name_base + '2b')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.Conv2D(nb_filter3, (1, 1), name=conv_name_base + + '2c', use_bias=use_bias)(x) + x = BatchNorm(name=bn_name_base + '2c')(x, training=train_bn) + + shortcut = KL.Conv2D(nb_filter3, (1, 1), strides=strides, + name=conv_name_base + '1', use_bias=use_bias)(input_tensor) + shortcut = BatchNorm(name=bn_name_base + '1')(shortcut, training=train_bn) + + x = KL.Add()([x, shortcut]) + x = KL.Activation('relu', name='res' + str(stage) + block + '_out')(x) + return x + + +def resnet_graph(input_image, architecture, stage5=False, train_bn=True): + """Build a ResNet graph. + architecture: Can be resnet50 or resnet101 + stage5: Boolean. If False, stage5 of the network is not created + train_bn: Boolean. Train or freeze Batch Norm layers + """ + assert architecture in ["resnet50", "resnet101"] + # Stage 1 + x = KL.ZeroPadding2D((3, 3))(input_image) + x = KL.Conv2D(64, (7, 7), strides=(2, 2), name='conv1', use_bias=True)(x) + x = BatchNorm(name='bn_conv1')(x, training=train_bn) + x = KL.Activation('relu')(x) + C1 = x = KL.MaxPooling2D((3, 3), strides=(2, 2), padding="same")(x) + # Stage 2 + x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1), train_bn=train_bn) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b', train_bn=train_bn) + C2 = x = identity_block(x, 3, [64, 64, 256], stage=2, block='c', train_bn=train_bn) + # Stage 3 + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a', train_bn=train_bn) + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b', train_bn=train_bn) + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c', train_bn=train_bn) + C3 = x = identity_block(x, 3, [128, 128, 512], stage=3, block='d', train_bn=train_bn) + # Stage 4 + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a', train_bn=train_bn) + block_count = {"resnet50": 5, "resnet101": 22}[architecture] + for i in range(block_count): + x = identity_block(x, 3, [256, 256, 1024], stage=4, block=chr(98 + i), train_bn=train_bn) + C4 = x + # Stage 5 + if stage5: + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a', train_bn=train_bn) + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b', train_bn=train_bn) + C5 = x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c', train_bn=train_bn) + else: + C5 = None + return [C1, C2, C3, C4, C5] + + +############################################################ +# Proposal Layer +############################################################ + +def apply_box_deltas_graph(boxes, deltas): + """Applies the given deltas to the given boxes. + boxes: [N, (y1, x1, y2, x2)] boxes to update + deltas: [N, (dy, dx, log(dh), log(dw))] refinements to apply + """ + # Convert to y, x, h, w + height = boxes[:, 2] - boxes[:, 0] + width = boxes[:, 3] - boxes[:, 1] + center_y = boxes[:, 0] + 0.5 * height + center_x = boxes[:, 1] + 0.5 * width + # Apply deltas + center_y += deltas[:, 0] * height + center_x += deltas[:, 1] * width + height *= tf.exp(deltas[:, 2]) + width *= tf.exp(deltas[:, 3]) + # Convert back to y1, x1, y2, x2 + y1 = center_y - 0.5 * height + x1 = center_x - 0.5 * width + y2 = y1 + height + x2 = x1 + width + result = tf.stack([y1, x1, y2, x2], axis=1, name="apply_box_deltas_out") + return result + + +def clip_boxes_graph(boxes, window): + """ + boxes: [N, (y1, x1, y2, x2)] + window: [4] in the form y1, x1, y2, x2 + """ + # Split + wy1, wx1, wy2, wx2 = tf.split(window, 4) + y1, x1, y2, x2 = tf.split(boxes, 4, axis=1) + # Clip + y1 = tf.maximum(tf.minimum(y1, wy2), wy1) + x1 = tf.maximum(tf.minimum(x1, wx2), wx1) + y2 = tf.maximum(tf.minimum(y2, wy2), wy1) + x2 = tf.maximum(tf.minimum(x2, wx2), wx1) + clipped = tf.concat([y1, x1, y2, x2], axis=1, name="clipped_boxes") + clipped.set_shape((clipped.shape[0], 4)) + return clipped + + +class ProposalLayer(KE.Layer): + """Receives anchor scores and selects a subset to pass as proposals + to the second stage. Filtering is done based on anchor scores and + non-max suppression to remove overlaps. It also applies bounding + box refinement deltas to anchors. + + Inputs: + rpn_probs: [batch, anchors, (bg prob, fg prob)] + rpn_bbox: [batch, anchors, (dy, dx, log(dh), log(dw))] + anchors: [batch, (y1, x1, y2, x2)] anchors in normalized coordinates + + Returns: + Proposals in normalized coordinates [batch, rois, (y1, x1, y2, x2)] + """ + + def __init__(self, proposal_count, nms_threshold, config=None, **kwargs): + super(ProposalLayer, self).__init__(**kwargs) + self.config = config + self.proposal_count = proposal_count + self.nms_threshold = nms_threshold + + def call(self, inputs): + # Box Scores. Use the foreground class confidence. [Batch, num_rois, 1] + scores = inputs[0][:, :, 1] + # Box deltas [batch, num_rois, 4] + deltas = inputs[1] + deltas = deltas * np.reshape(self.config.RPN_BBOX_STD_DEV, [1, 1, 4]) + # Anchors + anchors = inputs[2] + + # Improve performance by trimming to top anchors by score + # and doing the rest on the smaller subset. + pre_nms_limit = tf.minimum(6000, tf.shape(anchors)[1]) + ix = tf.nn.top_k(scores, pre_nms_limit, sorted=True, + name="top_anchors").indices + scores = utils.batch_slice([scores, ix], lambda x, y: tf.gather(x, y), + self.config.IMAGES_PER_GPU) + deltas = utils.batch_slice([deltas, ix], lambda x, y: tf.gather(x, y), + self.config.IMAGES_PER_GPU) + pre_nms_anchors = utils.batch_slice([anchors, ix], lambda a, x: tf.gather(a, x), + self.config.IMAGES_PER_GPU, + names=["pre_nms_anchors"]) + + # Apply deltas to anchors to get refined anchors. + # [batch, N, (y1, x1, y2, x2)] + boxes = utils.batch_slice([pre_nms_anchors, deltas], + lambda x, y: apply_box_deltas_graph(x, y), + self.config.IMAGES_PER_GPU, + names=["refined_anchors"]) + + # Clip to image boundaries. Since we're in normalized coordinates, + # clip to 0..1 range. [batch, N, (y1, x1, y2, x2)] + window = np.array([0, 0, 1, 1], dtype=np.float32) + boxes = utils.batch_slice(boxes, + lambda x: clip_boxes_graph(x, window), + self.config.IMAGES_PER_GPU, + names=["refined_anchors_clipped"]) + + # Filter out small boxes + # According to Xinlei Chen's paper, this reduces detection accuracy + # for small objects, so we're skipping it. + + # Non-max suppression + def nms(boxes, scores): + indices = tf.image.non_max_suppression( + boxes, scores, self.proposal_count, + self.nms_threshold, name="rpn_non_max_suppression") + proposals = tf.gather(boxes, indices) + # Pad if needed + padding = tf.maximum(self.proposal_count - tf.shape(proposals)[0], 0) + proposals = tf.pad(proposals, [(0, padding), (0, 0)]) + return proposals + + proposals = utils.batch_slice([boxes, scores], nms, + self.config.IMAGES_PER_GPU) + return proposals + + def compute_output_shape(self, input_shape): + return (None, self.proposal_count, 4) + + +############################################################ +# ROIAlign Layer +############################################################ + +def log2_graph(x): + """Implementation of Log2. TF doesn't have a native implementation.""" + return tf.log(x) / tf.log(2.0) + + +class PyramidROIAlign(KE.Layer): + """Implements ROI Pooling on multiple levels of the feature pyramid. + + Params: + - pool_shape: [height, width] of the output pooled regions. Usually [7, 7] + + Inputs: + - boxes: [batch, num_boxes, (y1, x1, y2, x2)] in normalized + coordinates. Possibly padded with zeros if not enough + boxes to fill the array. + - image_meta: [batch, (meta data)] Image details. See compose_image_meta() + - Feature maps: List of feature maps from different levels of the pyramid. + Each is [batch, height, width, channels] + + Output: + Pooled regions in the shape: [batch, num_boxes, height, width, channels]. + The width and height are those specific in the pool_shape in the layer + constructor. + """ + + def __init__(self, pool_shape, **kwargs): + super(PyramidROIAlign, self).__init__(**kwargs) + self.pool_shape = tuple(pool_shape) + + def call(self, inputs): + # Crop boxes [batch, num_boxes, (y1, x1, y2, x2)] in normalized coords + boxes = inputs[0] + + # Image meta + # Holds details about the image. See compose_image_meta() + image_meta = inputs[1] + + # Feature Maps. List of feature maps from different level of the + # feature pyramid. Each is [batch, height, width, channels] + feature_maps = inputs[2:] + + # Assign each ROI to a level in the pyramid based on the ROI area. + y1, x1, y2, x2 = tf.split(boxes, 4, axis=2) + h = y2 - y1 + w = x2 - x1 + # Use shape of first image. Images in a batch must have the same size. + image_shape = parse_image_meta_graph(image_meta)['image_shape'][0] + # Equation 1 in the Feature Pyramid Networks paper. Account for + # the fact that our coordinates are normalized here. + # e.g. a 224x224 ROI (in pixels) maps to P4 + image_area = tf.cast(image_shape[0] * image_shape[1], tf.float32) + roi_level = log2_graph(tf.sqrt(h * w) / (224.0 / tf.sqrt(image_area))) + roi_level = tf.minimum(5, tf.maximum( + 2, 4 + tf.cast(tf.round(roi_level), tf.int32))) + roi_level = tf.squeeze(roi_level, 2) + + # Loop through levels and apply ROI pooling to each. P2 to P5. + pooled = [] + box_to_level = [] + for i, level in enumerate(range(2, 6)): + ix = tf.where(tf.equal(roi_level, level)) + level_boxes = tf.gather_nd(boxes, ix) + + # Box indices for crop_and_resize. + box_indices = tf.cast(ix[:, 0], tf.int32) + + # Keep track of which box is mapped to which level + box_to_level.append(ix) + + # Stop gradient propogation to ROI proposals + level_boxes = tf.stop_gradient(level_boxes) + box_indices = tf.stop_gradient(box_indices) + + # Crop and Resize + # From Mask R-CNN paper: "We sample four regular locations, so + # that we can evaluate either max or average pooling. In fact, + # interpolating only a single value at each bin center (without + # pooling) is nearly as effective." + # + # Here we use the simplified approach of a single value per bin, + # which is how it's done in tf.crop_and_resize() + # Result: [batch * num_boxes, pool_height, pool_width, channels] + pooled.append(tf.image.crop_and_resize( + feature_maps[i], level_boxes, box_indices, self.pool_shape, + method="bilinear")) + + # Pack pooled features into one tensor + pooled = tf.concat(pooled, axis=0) + + # Pack box_to_level mapping into one array and add another + # column representing the order of pooled boxes + box_to_level = tf.concat(box_to_level, axis=0) + box_range = tf.expand_dims(tf.range(tf.shape(box_to_level)[0]), 1) + box_to_level = tf.concat([tf.cast(box_to_level, tf.int32), box_range], + axis=1) + + # Rearrange pooled features to match the order of the original boxes + # Sort box_to_level by batch then box index + # TF doesn't have a way to sort by two columns, so merge them and sort. + sorting_tensor = box_to_level[:, 0] * 100000 + box_to_level[:, 1] + ix = tf.nn.top_k(sorting_tensor, k=tf.shape( + box_to_level)[0]).indices[::-1] + ix = tf.gather(box_to_level[:, 2], ix) + pooled = tf.gather(pooled, ix) + + # Re-add the batch dimension + pooled = tf.expand_dims(pooled, 0) + return pooled + + def compute_output_shape(self, input_shape): + return input_shape[0][:2] + self.pool_shape + (input_shape[2][-1],) + + +############################################################ +# Detection Target Layer +############################################################ + +def overlaps_graph(boxes1, boxes2): + """Computes IoU overlaps between two sets of boxes. + boxes1, boxes2: [N, (y1, x1, y2, x2)]. + """ + # 1. Tile boxes2 and repeat boxes1. This allows us to compare + # every boxes1 against every boxes2 without loops. + # TF doesn't have an equivalent to np.repeat() so simulate it + # using tf.tile() and tf.reshape. + b1 = tf.reshape(tf.tile(tf.expand_dims(boxes1, 1), + [1, 1, tf.shape(boxes2)[0]]), [-1, 4]) + b2 = tf.tile(boxes2, [tf.shape(boxes1)[0], 1]) + # 2. Compute intersections + b1_y1, b1_x1, b1_y2, b1_x2 = tf.split(b1, 4, axis=1) + b2_y1, b2_x1, b2_y2, b2_x2 = tf.split(b2, 4, axis=1) + y1 = tf.maximum(b1_y1, b2_y1) + x1 = tf.maximum(b1_x1, b2_x1) + y2 = tf.minimum(b1_y2, b2_y2) + x2 = tf.minimum(b1_x2, b2_x2) + intersection = tf.maximum(x2 - x1, 0) * tf.maximum(y2 - y1, 0) + # 3. Compute unions + b1_area = (b1_y2 - b1_y1) * (b1_x2 - b1_x1) + b2_area = (b2_y2 - b2_y1) * (b2_x2 - b2_x1) + union = b1_area + b2_area - intersection + # 4. Compute IoU and reshape to [boxes1, boxes2] + iou = intersection / union + overlaps = tf.reshape(iou, [tf.shape(boxes1)[0], tf.shape(boxes2)[0]]) + return overlaps + + +def detection_targets_graph(proposals, gt_class_ids, gt_boxes, gt_masks, config): + """Generates detection targets for one image. Subsamples proposals and + generates target class IDs, bounding box deltas, and masks for each. + + Inputs: + proposals: [N, (y1, x1, y2, x2)] in normalized coordinates. Might + be zero padded if there are not enough proposals. + gt_class_ids: [MAX_GT_INSTANCES] int class IDs + gt_boxes: [MAX_GT_INSTANCES, (y1, x1, y2, x2)] in normalized coordinates. + gt_masks: [height, width, MAX_GT_INSTANCES] of boolean type. + + Returns: Target ROIs and corresponding class IDs, bounding box shifts, + and masks. + rois: [TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)] in normalized coordinates + class_ids: [TRAIN_ROIS_PER_IMAGE]. Integer class IDs. Zero padded. + deltas: [TRAIN_ROIS_PER_IMAGE, NUM_CLASSES, (dy, dx, log(dh), log(dw))] + Class-specific bbox refinements. + masks: [TRAIN_ROIS_PER_IMAGE, height, width). Masks cropped to bbox + boundaries and resized to neural network output size. + + Note: Returned arrays might be zero padded if not enough target ROIs. + """ + # Assertions + asserts = [ + tf.Assert(tf.greater(tf.shape(proposals)[0], 0), [proposals], + name="roi_assertion"), + ] + with tf.control_dependencies(asserts): + proposals = tf.identity(proposals) + + # Remove zero padding + proposals, _ = trim_zeros_graph(proposals, name="trim_proposals") + gt_boxes, non_zeros = trim_zeros_graph(gt_boxes, name="trim_gt_boxes") + gt_class_ids = tf.boolean_mask(gt_class_ids, non_zeros, + name="trim_gt_class_ids") + gt_masks = tf.gather(gt_masks, tf.where(non_zeros)[:, 0], axis=2, + name="trim_gt_masks") + + # Handle COCO crowds + # A crowd box in COCO is a bounding box around several instances. Exclude + # them from training. A crowd box is given a negative class ID. + crowd_ix = tf.where(gt_class_ids < 0)[:, 0] + non_crowd_ix = tf.where(gt_class_ids > 0)[:, 0] + crowd_boxes = tf.gather(gt_boxes, crowd_ix) + crowd_masks = tf.gather(gt_masks, crowd_ix, axis=2) + gt_class_ids = tf.gather(gt_class_ids, non_crowd_ix) + gt_boxes = tf.gather(gt_boxes, non_crowd_ix) + gt_masks = tf.gather(gt_masks, non_crowd_ix, axis=2) + + # Compute overlaps matrix [proposals, gt_boxes] + overlaps = overlaps_graph(proposals, gt_boxes) + + # Compute overlaps with crowd boxes [anchors, crowds] + crowd_overlaps = overlaps_graph(proposals, crowd_boxes) + crowd_iou_max = tf.reduce_max(crowd_overlaps, axis=1) + no_crowd_bool = (crowd_iou_max < 0.001) + + # Determine positive and negative ROIs + roi_iou_max = tf.reduce_max(overlaps, axis=1) + # 1. Positive ROIs are those with >= 0.5 IoU with a GT box + positive_roi_bool = (roi_iou_max >= 0.5) + positive_indices = tf.where(positive_roi_bool)[:, 0] + # 2. Negative ROIs are those with < 0.5 with every GT box. Skip crowds. + negative_indices = tf.where(tf.logical_and(roi_iou_max < 0.5, no_crowd_bool))[:, 0] + + # Subsample ROIs. Aim for 33% positive + # Positive ROIs + positive_count = int(config.TRAIN_ROIS_PER_IMAGE * + config.ROI_POSITIVE_RATIO) + positive_indices = tf.random_shuffle(positive_indices)[:positive_count] + positive_count = tf.shape(positive_indices)[0] + # Negative ROIs. Add enough to maintain positive:negative ratio. + r = 1.0 / config.ROI_POSITIVE_RATIO + negative_count = tf.cast(r * tf.cast(positive_count, tf.float32), tf.int32) - positive_count + negative_indices = tf.random_shuffle(negative_indices)[:negative_count] + # Gather selected ROIs + positive_rois = tf.gather(proposals, positive_indices) + negative_rois = tf.gather(proposals, negative_indices) + + # Assign positive ROIs to GT boxes. + positive_overlaps = tf.gather(overlaps, positive_indices) + roi_gt_box_assignment = tf.cond( + tf.greater(tf.shape(positive_overlaps)[1], 0), + true_fn=lambda: tf.argmax(positive_overlaps, axis=1), + false_fn=lambda: tf.cast(tf.constant([]), tf.int64) + ) + roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment) + roi_gt_class_ids = tf.gather(gt_class_ids, roi_gt_box_assignment) + + # Compute bbox refinement for positive ROIs + deltas = utils.box_refinement_graph(positive_rois, roi_gt_boxes) + deltas /= config.BBOX_STD_DEV + + # Assign positive ROIs to GT masks + # Permute masks to [N, height, width, 1] + transposed_masks = tf.expand_dims(tf.transpose(gt_masks, [2, 0, 1]), -1) + # Pick the right mask for each ROI + roi_masks = tf.gather(transposed_masks, roi_gt_box_assignment) + + # Compute mask targets + boxes = positive_rois + if config.USE_MINI_MASK: + # Transform ROI coordinates from normalized image space + # to normalized mini-mask space. + y1, x1, y2, x2 = tf.split(positive_rois, 4, axis=1) + gt_y1, gt_x1, gt_y2, gt_x2 = tf.split(roi_gt_boxes, 4, axis=1) + gt_h = gt_y2 - gt_y1 + gt_w = gt_x2 - gt_x1 + y1 = (y1 - gt_y1) / gt_h + x1 = (x1 - gt_x1) / gt_w + y2 = (y2 - gt_y1) / gt_h + x2 = (x2 - gt_x1) / gt_w + boxes = tf.concat([y1, x1, y2, x2], 1) + box_ids = tf.range(0, tf.shape(roi_masks)[0]) + masks = tf.image.crop_and_resize(tf.cast(roi_masks, tf.float32), boxes, + box_ids, + config.MASK_SHAPE) + # Remove the extra dimension from masks. + masks = tf.squeeze(masks, axis=3) + + # Threshold mask pixels at 0.5 to have GT masks be 0 or 1 to use with + # binary cross entropy loss. + masks = tf.round(masks) + + # Append negative ROIs and pad bbox deltas and masks that + # are not used for negative ROIs with zeros. + rois = tf.concat([positive_rois, negative_rois], axis=0) + N = tf.shape(negative_rois)[0] + P = tf.maximum(config.TRAIN_ROIS_PER_IMAGE - tf.shape(rois)[0], 0) + rois = tf.pad(rois, [(0, P), (0, 0)]) + roi_gt_boxes = tf.pad(roi_gt_boxes, [(0, N + P), (0, 0)]) + roi_gt_class_ids = tf.pad(roi_gt_class_ids, [(0, N + P)]) + deltas = tf.pad(deltas, [(0, N + P), (0, 0)]) + masks = tf.pad(masks, [[0, N + P], (0, 0), (0, 0)]) + + return rois, roi_gt_class_ids, deltas, masks + + +class DetectionTargetLayer(KE.Layer): + """Subsamples proposals and generates target box refinement, class_ids, + and masks for each. + + Inputs: + proposals: [batch, N, (y1, x1, y2, x2)] in normalized coordinates. Might + be zero padded if there are not enough proposals. + gt_class_ids: [batch, MAX_GT_INSTANCES] Integer class IDs. + gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] in normalized + coordinates. + gt_masks: [batch, height, width, MAX_GT_INSTANCES] of boolean type + + Returns: Target ROIs and corresponding class IDs, bounding box shifts, + and masks. + rois: [batch, TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)] in normalized + coordinates + target_class_ids: [batch, TRAIN_ROIS_PER_IMAGE]. Integer class IDs. + target_deltas: [batch, TRAIN_ROIS_PER_IMAGE, NUM_CLASSES, + (dy, dx, log(dh), log(dw), class_id)] + Class-specific bbox refinements. + target_mask: [batch, TRAIN_ROIS_PER_IMAGE, height, width) + Masks cropped to bbox boundaries and resized to neural + network output size. + + Note: Returned arrays might be zero padded if not enough target ROIs. + """ + + def __init__(self, config, **kwargs): + super(DetectionTargetLayer, self).__init__(**kwargs) + self.config = config + + def call(self, inputs): + proposals = inputs[0] + gt_class_ids = inputs[1] + gt_boxes = inputs[2] + gt_masks = inputs[3] + + # Slice the batch and run a graph for each slice + # TODO: Rename target_bbox to target_deltas for clarity + names = ["rois", "target_class_ids", "target_bbox", "target_mask"] + outputs = utils.batch_slice( + [proposals, gt_class_ids, gt_boxes, gt_masks], + lambda w, x, y, z: detection_targets_graph( + w, x, y, z, self.config), + self.config.IMAGES_PER_GPU, names=names) + return outputs + + def compute_output_shape(self, input_shape): + return [ + (None, self.config.TRAIN_ROIS_PER_IMAGE, 4), # rois + (None, 1), # class_ids + (None, self.config.TRAIN_ROIS_PER_IMAGE, 4), # deltas + (None, self.config.TRAIN_ROIS_PER_IMAGE, self.config.MASK_SHAPE[0], + self.config.MASK_SHAPE[1]) # masks + ] + + def compute_mask(self, inputs, mask=None): + return [None, None, None, None] + + +############################################################ +# Detection Layer +############################################################ + +def refine_detections_graph(rois, probs, deltas, window, config): + """Refine classified proposals and filter overlaps and return final + detections. + + Inputs: + rois: [N, (y1, x1, y2, x2)] in normalized coordinates + probs: [N, num_classes]. Class probabilities. + deltas: [N, num_classes, (dy, dx, log(dh), log(dw))]. Class-specific + bounding box deltas. + window: (y1, x1, y2, x2) in image coordinates. The part of the image + that contains the image excluding the padding. + + Returns detections shaped: [N, (y1, x1, y2, x2, class_id, score)] where + coordinates are normalized. + """ + # Class IDs per ROI + class_ids = tf.argmax(probs, axis=1, output_type=tf.int32) + # Class probability of the top class of each ROI + indices = tf.stack([tf.range(probs.shape[0]), class_ids], axis=1) + class_scores = tf.gather_nd(probs, indices) + # Class-specific bounding box deltas + deltas_specific = tf.gather_nd(deltas, indices) + # Apply bounding box deltas + # Shape: [boxes, (y1, x1, y2, x2)] in normalized coordinates + refined_rois = apply_box_deltas_graph( + rois, deltas_specific * config.BBOX_STD_DEV) + # Clip boxes to image window + refined_rois = clip_boxes_graph(refined_rois, window) + + # TODO: Filter out boxes with zero area + + # Filter out background boxes + keep = tf.where(class_ids > 0)[:, 0] + # Filter out low confidence boxes + if config.DETECTION_MIN_CONFIDENCE: + conf_keep = tf.where(class_scores >= config.DETECTION_MIN_CONFIDENCE)[:, 0] + keep = tf.sets.set_intersection(tf.expand_dims(keep, 0), + tf.expand_dims(conf_keep, 0)) + keep = tf.sparse_tensor_to_dense(keep)[0] + + # Apply per-class NMS + # 1. Prepare variables + pre_nms_class_ids = tf.gather(class_ids, keep) + pre_nms_scores = tf.gather(class_scores, keep) + pre_nms_rois = tf.gather(refined_rois, keep) + unique_pre_nms_class_ids = tf.unique(pre_nms_class_ids)[0] + + def nms_keep_map(class_id): + """Apply Non-Maximum Suppression on ROIs of the given class.""" + # Indices of ROIs of the given class + ixs = tf.where(tf.equal(pre_nms_class_ids, class_id))[:, 0] + # Apply NMS + class_keep = tf.image.non_max_suppression( + tf.gather(pre_nms_rois, ixs), + tf.gather(pre_nms_scores, ixs), + max_output_size=config.DETECTION_MAX_INSTANCES, + iou_threshold=config.DETECTION_NMS_THRESHOLD) + # Map indices + class_keep = tf.gather(keep, tf.gather(ixs, class_keep)) + # Pad with -1 so returned tensors have the same shape + gap = config.DETECTION_MAX_INSTANCES - tf.shape(class_keep)[0] + class_keep = tf.pad(class_keep, [(0, gap)], + mode='CONSTANT', constant_values=-1) + # Set shape so map_fn() can infer result shape + class_keep.set_shape([config.DETECTION_MAX_INSTANCES]) + return class_keep + + # 2. Map over class IDs + nms_keep = tf.map_fn(nms_keep_map, unique_pre_nms_class_ids, + dtype=tf.int64) + # 3. Merge results into one list, and remove -1 padding + nms_keep = tf.reshape(nms_keep, [-1]) + nms_keep = tf.gather(nms_keep, tf.where(nms_keep > -1)[:, 0]) + # 4. Compute intersection between keep and nms_keep + keep = tf.sets.set_intersection(tf.expand_dims(keep, 0), + tf.expand_dims(nms_keep, 0)) + keep = tf.sparse_tensor_to_dense(keep)[0] + # Keep top detections + roi_count = config.DETECTION_MAX_INSTANCES + class_scores_keep = tf.gather(class_scores, keep) + num_keep = tf.minimum(tf.shape(class_scores_keep)[0], roi_count) + top_ids = tf.nn.top_k(class_scores_keep, k=num_keep, sorted=True)[1] + keep = tf.gather(keep, top_ids) + + # Arrange output as [N, (y1, x1, y2, x2, class_id, score)] + # Coordinates are normalized. + detections = tf.concat([ + tf.gather(refined_rois, keep), + tf.to_float(tf.gather(class_ids, keep))[..., tf.newaxis], + tf.gather(class_scores, keep)[..., tf.newaxis] + ], axis=1) + + # Pad with zeros if detections < DETECTION_MAX_INSTANCES + gap = config.DETECTION_MAX_INSTANCES - tf.shape(detections)[0] + detections = tf.pad(detections, [(0, gap), (0, 0)], "CONSTANT") + return detections + + +class DetectionLayer(KE.Layer): + """Takes classified proposal boxes and their bounding box deltas and + returns the final detection boxes. + + Returns: + [batch, num_detections, (y1, x1, y2, x2, class_id, class_score)] where + coordinates are normalized. + """ + + def __init__(self, config=None, **kwargs): + super(DetectionLayer, self).__init__(**kwargs) + self.config = config + + def call(self, inputs): + rois = inputs[0] + mrcnn_class = inputs[1] + mrcnn_bbox = inputs[2] + image_meta = inputs[3] + + # Get windows of images in normalized coordinates. Windows are the area + # in the image that excludes the padding. + # Use the shape of the first image in the batch to normalize the window + # because we know that all images get resized to the same size. + m = parse_image_meta_graph(image_meta) + image_shape = m['image_shape'][0] + window = norm_boxes_graph(m['window'], image_shape[:2]) + + # Run detection refinement graph on each item in the batch + detections_batch = utils.batch_slice( + [rois, mrcnn_class, mrcnn_bbox, window], + lambda x, y, w, z: refine_detections_graph(x, y, w, z, self.config), + self.config.IMAGES_PER_GPU) + + # Reshape output + # [batch, num_detections, (y1, x1, y2, x2, class_score)] in + # normalized coordinates + return tf.reshape( + detections_batch, + [self.config.BATCH_SIZE, self.config.DETECTION_MAX_INSTANCES, 6]) + + def compute_output_shape(self, input_shape): + return (None, self.config.DETECTION_MAX_INSTANCES, 6) + + +############################################################ +# Region Proposal Network (RPN) +############################################################ + +def rpn_graph(feature_map, anchors_per_location, anchor_stride): + """Builds the computation graph of Region Proposal Network. + + feature_map: backbone features [batch, height, width, depth] + anchors_per_location: number of anchors per pixel in the feature map + anchor_stride: Controls the density of anchors. Typically 1 (anchors for + every pixel in the feature map), or 2 (every other pixel). + + Returns: + rpn_logits: [batch, H, W, 2] Anchor classifier logits (before softmax) + rpn_probs: [batch, H, W, 2] Anchor classifier probabilities. + rpn_bbox: [batch, H, W, (dy, dx, log(dh), log(dw))] Deltas to be + applied to anchors. + """ + # TODO: check if stride of 2 causes alignment issues if the feature map + # is not even. + # Shared convolutional base of the RPN + shared = KL.Conv2D(512, (3, 3), padding='same', activation='relu', + strides=anchor_stride, + name='rpn_conv_shared')(feature_map) + + # Anchor Score. [batch, height, width, anchors per location * 2]. + x = KL.Conv2D(2 * anchors_per_location, (1, 1), padding='valid', + activation='linear', name='rpn_class_raw')(shared) + + # Reshape to [batch, anchors, 2] + rpn_class_logits = KL.Lambda( + lambda t: tf.reshape(t, [tf.shape(t)[0], -1, 2]))(x) + + # Softmax on last dimension of BG/FG. + rpn_probs = KL.Activation( + "softmax", name="rpn_class_xxx")(rpn_class_logits) + + # Bounding box refinement. [batch, H, W, anchors per location, depth] + # where depth is [x, y, log(w), log(h)] + x = KL.Conv2D(anchors_per_location * 4, (1, 1), padding="valid", + activation='linear', name='rpn_bbox_pred')(shared) + + # Reshape to [batch, anchors, 4] + rpn_bbox = KL.Lambda(lambda t: tf.reshape(t, [tf.shape(t)[0], -1, 4]))(x) + + return [rpn_class_logits, rpn_probs, rpn_bbox] + + +def build_rpn_model(anchor_stride, anchors_per_location, depth): + """Builds a Keras model of the Region Proposal Network. + It wraps the RPN graph so it can be used multiple times with shared + weights. + + anchors_per_location: number of anchors per pixel in the feature map + anchor_stride: Controls the density of anchors. Typically 1 (anchors for + every pixel in the feature map), or 2 (every other pixel). + depth: Depth of the backbone feature map. + + Returns a Keras Model object. The model outputs, when called, are: + rpn_logits: [batch, H, W, 2] Anchor classifier logits (before softmax) + rpn_probs: [batch, W, W, 2] Anchor classifier probabilities. + rpn_bbox: [batch, H, W, (dy, dx, log(dh), log(dw))] Deltas to be + applied to anchors. + """ + input_feature_map = KL.Input(shape=[None, None, depth], + name="input_rpn_feature_map") + outputs = rpn_graph(input_feature_map, anchors_per_location, anchor_stride) + return KM.Model([input_feature_map], outputs, name="rpn_model") + + +############################################################ +# Feature Pyramid Network Heads +############################################################ + +def fpn_classifier_graph(rois, feature_maps, image_meta, + pool_size, num_classes, train_bn=True, + fc_layers_size=1024): + """Builds the computation graph of the feature pyramid network classifier + and regressor heads. + + rois: [batch, num_rois, (y1, x1, y2, x2)] Proposal boxes in normalized + coordinates. + feature_maps: List of feature maps from different layers of the pyramid, + [P2, P3, P4, P5]. Each has a different resolution. + - image_meta: [batch, (meta data)] Image details. See compose_image_meta() + pool_size: The width of the square feature map generated from ROI Pooling. + num_classes: number of classes, which determines the depth of the results + train_bn: Boolean. Train or freeze Batch Norm layers + fc_layers_size: Size of the 2 FC layers + + Returns: + logits: [N, NUM_CLASSES] classifier logits (before softmax) + probs: [N, NUM_CLASSES] classifier probabilities + bbox_deltas: [N, (dy, dx, log(dh), log(dw))] Deltas to apply to + proposal boxes + """ + # ROI Pooling + # Shape: [batch, num_boxes, pool_height, pool_width, channels] + x = PyramidROIAlign([pool_size, pool_size], + name="roi_align_classifier")([rois, image_meta] + feature_maps) + # Two 1024 FC layers (implemented with Conv2D for consistency) + x = KL.TimeDistributed(KL.Conv2D(fc_layers_size, (pool_size, pool_size), padding="valid"), + name="mrcnn_class_conv1")(x) + x = KL.TimeDistributed(BatchNorm(), name='mrcnn_class_bn1')(x, training=train_bn) + x = KL.Activation('relu')(x) + x = KL.TimeDistributed(KL.Conv2D(fc_layers_size, (1, 1)), + name="mrcnn_class_conv2")(x) + x = KL.TimeDistributed(BatchNorm(), name='mrcnn_class_bn2')(x, training=train_bn) + x = KL.Activation('relu')(x) + + shared = KL.Lambda(lambda x: K.squeeze(K.squeeze(x, 3), 2), + name="pool_squeeze")(x) + + # Classifier head + mrcnn_class_logits = KL.TimeDistributed(KL.Dense(num_classes), + name='mrcnn_class_logits')(shared) + mrcnn_probs = KL.TimeDistributed(KL.Activation("softmax"), + name="mrcnn_class")(mrcnn_class_logits) + + # BBox head + # [batch, boxes, num_classes * (dy, dx, log(dh), log(dw))] + x = KL.TimeDistributed(KL.Dense(num_classes * 4, activation='linear'), + name='mrcnn_bbox_fc')(shared) + # Reshape to [batch, boxes, num_classes, (dy, dx, log(dh), log(dw))] + s = K.int_shape(x) + mrcnn_bbox = KL.Reshape((s[1], num_classes, 4), name="mrcnn_bbox")(x) + + return mrcnn_class_logits, mrcnn_probs, mrcnn_bbox + + +def build_fpn_mask_graph(rois, feature_maps, image_meta, + pool_size, num_classes, train_bn=True): + """Builds the computation graph of the mask head of Feature Pyramid Network. + + rois: [batch, num_rois, (y1, x1, y2, x2)] Proposal boxes in normalized + coordinates. + feature_maps: List of feature maps from different layers of the pyramid, + [P2, P3, P4, P5]. Each has a different resolution. + image_meta: [batch, (meta data)] Image details. See compose_image_meta() + pool_size: The width of the square feature map generated from ROI Pooling. + num_classes: number of classes, which determines the depth of the results + train_bn: Boolean. Train or freeze Batch Norm layers + + Returns: Masks [batch, roi_count, height, width, num_classes] + """ + # ROI Pooling + # Shape: [batch, boxes, pool_height, pool_width, channels] + x = PyramidROIAlign([pool_size, pool_size], + name="roi_align_mask")([rois, image_meta] + feature_maps) + + # Conv layers + x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"), + name="mrcnn_mask_conv1")(x) + x = KL.TimeDistributed(BatchNorm(), + name='mrcnn_mask_bn1')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"), + name="mrcnn_mask_conv2")(x) + x = KL.TimeDistributed(BatchNorm(), + name='mrcnn_mask_bn2')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"), + name="mrcnn_mask_conv3")(x) + x = KL.TimeDistributed(BatchNorm(), + name='mrcnn_mask_bn3')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"), + name="mrcnn_mask_conv4")(x) + x = KL.TimeDistributed(BatchNorm(), + name='mrcnn_mask_bn4')(x, training=train_bn) + x = KL.Activation('relu')(x) + + x = KL.TimeDistributed(KL.Conv2DTranspose(256, (2, 2), strides=2, activation="relu"), + name="mrcnn_mask_deconv")(x) + x = KL.TimeDistributed(KL.Conv2D(num_classes, (1, 1), strides=1, activation="sigmoid"), + name="mrcnn_mask")(x) + return x + + +############################################################ +# Loss Functions +############################################################ + +def smooth_l1_loss(y_true, y_pred): + """Implements Smooth-L1 loss. + y_true and y_pred are typically: [N, 4], but could be any shape. + """ + diff = K.abs(y_true - y_pred) + less_than_one = K.cast(K.less(diff, 1.0), "float32") + loss = (less_than_one * 0.5 * diff ** 2) + (1 - less_than_one) * (diff - 0.5) + return loss + + +def rpn_class_loss_graph(rpn_match, rpn_class_logits): + """RPN anchor classifier loss. + + rpn_match: [batch, anchors, 1]. Anchor match type. 1=positive, + -1=negative, 0=neutral anchor. + rpn_class_logits: [batch, anchors, 2]. RPN classifier logits for FG/BG. + """ + # Squeeze last dim to simplify + rpn_match = tf.squeeze(rpn_match, -1) + # Get anchor classes. Convert the -1/+1 match to 0/1 values. + anchor_class = K.cast(K.equal(rpn_match, 1), tf.int32) + # Positive and Negative anchors contribute to the loss, + # but neutral anchors (match value = 0) don't. + indices = tf.where(K.not_equal(rpn_match, 0)) + # Pick rows that contribute to the loss and filter out the rest. + rpn_class_logits = tf.gather_nd(rpn_class_logits, indices) + anchor_class = tf.gather_nd(anchor_class, indices) + # Cross entropy loss + loss = K.sparse_categorical_crossentropy(target=anchor_class, + output=rpn_class_logits, + from_logits=True) + loss = K.switch(tf.size(loss) > 0, K.mean(loss), tf.constant(0.0)) + return loss + + +def rpn_bbox_loss_graph(config, target_bbox, rpn_match, rpn_bbox): + """Return the RPN bounding box loss graph. + + config: the model config object. + target_bbox: [batch, max positive anchors, (dy, dx, log(dh), log(dw))]. + Uses 0 padding to fill in unsed bbox deltas. + rpn_match: [batch, anchors, 1]. Anchor match type. 1=positive, + -1=negative, 0=neutral anchor. + rpn_bbox: [batch, anchors, (dy, dx, log(dh), log(dw))] + """ + # Positive anchors contribute to the loss, but negative and + # neutral anchors (match value of 0 or -1) don't. + rpn_match = K.squeeze(rpn_match, -1) + indices = tf.where(K.equal(rpn_match, 1)) + + # Pick bbox deltas that contribute to the loss + rpn_bbox = tf.gather_nd(rpn_bbox, indices) + + # Trim target bounding box deltas to the same length as rpn_bbox. + batch_counts = K.sum(K.cast(K.equal(rpn_match, 1), tf.int32), axis=1) + target_bbox = batch_pack_graph(target_bbox, batch_counts, + config.IMAGES_PER_GPU) + + # TODO: use smooth_l1_loss() rather than reimplementing here + # to reduce code duplication + diff = K.abs(target_bbox - rpn_bbox) + less_than_one = K.cast(K.less(diff, 1.0), "float32") + loss = (less_than_one * 0.5 * diff ** 2) + (1 - less_than_one) * (diff - 0.5) + + loss = K.switch(tf.size(loss) > 0, K.mean(loss), tf.constant(0.0)) + return loss + + +def mrcnn_class_loss_graph(target_class_ids, pred_class_logits, + active_class_ids): + """Loss for the classifier head of Mask RCNN. + + target_class_ids: [batch, num_rois]. Integer class IDs. Uses zero + padding to fill in the array. + pred_class_logits: [batch, num_rois, num_classes] + active_class_ids: [batch, num_classes]. Has a value of 1 for + classes that are in the dataset of the image, and 0 + for classes that are not in the dataset. + """ + # During model building, Keras calls this function with + # target_class_ids of type float32. Unclear why. Cast it + # to int to get around it. + target_class_ids = tf.cast(target_class_ids, 'int64') + + # Find predictions of classes that are not in the dataset. + pred_class_ids = tf.argmax(pred_class_logits, axis=2) + # TODO: Update this line to work with batch > 1. Right now it assumes all + # images in a batch have the same active_class_ids + pred_active = tf.gather(active_class_ids[0], pred_class_ids) + + # Loss + loss = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=target_class_ids, logits=pred_class_logits) + + # Erase losses of predictions of classes that are not in the active + # classes of the image. + loss = loss * pred_active + + # Computer loss mean. Use only predictions that contribute + # to the loss to get a correct mean. + loss = tf.reduce_sum(loss) / tf.reduce_sum(pred_active) + return loss + + +def mrcnn_bbox_loss_graph(target_bbox, target_class_ids, pred_bbox): + """Loss for Mask R-CNN bounding box refinement. + + target_bbox: [batch, num_rois, (dy, dx, log(dh), log(dw))] + target_class_ids: [batch, num_rois]. Integer class IDs. + pred_bbox: [batch, num_rois, num_classes, (dy, dx, log(dh), log(dw))] + """ + # Reshape to merge batch and roi dimensions for simplicity. + target_class_ids = K.reshape(target_class_ids, (-1,)) + target_bbox = K.reshape(target_bbox, (-1, 4)) + pred_bbox = K.reshape(pred_bbox, (-1, K.int_shape(pred_bbox)[2], 4)) + + # Only positive ROIs contribute to the loss. And only + # the right class_id of each ROI. Get their indices. + positive_roi_ix = tf.where(target_class_ids > 0)[:, 0] + positive_roi_class_ids = tf.cast( + tf.gather(target_class_ids, positive_roi_ix), tf.int64) + indices = tf.stack([positive_roi_ix, positive_roi_class_ids], axis=1) + + # Gather the deltas (predicted and true) that contribute to loss + target_bbox = tf.gather(target_bbox, positive_roi_ix) + pred_bbox = tf.gather_nd(pred_bbox, indices) + + # Smooth-L1 Loss + loss = K.switch(tf.size(target_bbox) > 0, + smooth_l1_loss(y_true=target_bbox, y_pred=pred_bbox), + tf.constant(0.0)) + loss = K.mean(loss) + return loss + + +def mrcnn_mask_loss_graph(target_masks, target_class_ids, pred_masks): + """Mask binary cross-entropy loss for the masks head. + + target_masks: [batch, num_rois, height, width]. + A float32 tensor of values 0 or 1. Uses zero padding to fill array. + target_class_ids: [batch, num_rois]. Integer class IDs. Zero padded. + pred_masks: [batch, proposals, height, width, num_classes] float32 tensor + with values from 0 to 1. + """ + # Reshape for simplicity. Merge first two dimensions into one. + target_class_ids = K.reshape(target_class_ids, (-1,)) + mask_shape = tf.shape(target_masks) + target_masks = K.reshape(target_masks, (-1, mask_shape[2], mask_shape[3])) + pred_shape = tf.shape(pred_masks) + pred_masks = K.reshape(pred_masks, + (-1, pred_shape[2], pred_shape[3], pred_shape[4])) + # Permute predicted masks to [N, num_classes, height, width] + pred_masks = tf.transpose(pred_masks, [0, 3, 1, 2]) + + # Only positive ROIs contribute to the loss. And only + # the class specific mask of each ROI. + positive_ix = tf.where(target_class_ids > 0)[:, 0] + positive_class_ids = tf.cast( + tf.gather(target_class_ids, positive_ix), tf.int64) + indices = tf.stack([positive_ix, positive_class_ids], axis=1) + + # Gather the masks (predicted and true) that contribute to loss + y_true = tf.gather(target_masks, positive_ix) + y_pred = tf.gather_nd(pred_masks, indices) + + # Compute binary cross entropy. If no positive ROIs, then return 0. + # shape: [batch, roi, num_classes] + loss = K.switch(tf.size(y_true) > 0, + K.binary_crossentropy(target=y_true, output=y_pred), + tf.constant(0.0)) + loss = K.mean(loss) + return loss + + +############################################################ +# Data Generator +############################################################ + +def load_image_gt(dataset, config, image_id, augment=False, augmentation=None, + use_mini_mask=False): + """Load and return ground truth data for an image (image, mask, bounding boxes). + + augment: (deprecated. Use augmentation instead). If true, apply random + image augmentation. Currently, only horizontal flipping is offered. + augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) augmentation. + For example, passing imgaug.augmenters.Fliplr(0.5) flips images + right/left 50% of the time. + use_mini_mask: If False, returns full-size masks that are the same height + and width as the original image. These can be big, for example + 1024x1024x100 (for 100 instances). Mini masks are smaller, typically, + 224x224 and are generated by extracting the bounding box of the + object and resizing it to MINI_MASK_SHAPE. + + Returns: + image: [height, width, 3] + shape: the original shape of the image before resizing and cropping. + class_ids: [instance_count] Integer class IDs + bbox: [instance_count, (y1, x1, y2, x2)] + mask: [height, width, instance_count]. The height and width are those + of the image unless use_mini_mask is True, in which case they are + defined in MINI_MASK_SHAPE. + """ + # Load image and mask + image = dataset.load_image(image_id) + mask, class_ids = dataset.load_mask(image_id) + original_shape = image.shape + image, window, scale, padding, crop = utils.resize_image( + image, + min_dim=config.IMAGE_MIN_DIM, + min_scale=config.IMAGE_MIN_SCALE, + max_dim=config.IMAGE_MAX_DIM, + mode=config.IMAGE_RESIZE_MODE) + mask = utils.resize_mask(mask, scale, padding, crop) + + # Random horizontal flips. + # TODO: will be removed in a future update in favor of augmentation + if augment: + logging.warning("'augment' is deprecated. Use 'augmentation' instead.") + if random.randint(0, 1): + image = np.fliplr(image) + mask = np.fliplr(mask) + + # Augmentation + # This requires the imgaug lib (https://github.com/aleju/imgaug) + if augmentation: + import imgaug + + # Augmenters that are safe to apply to masks + # Some, such as Affine, have settings that make them unsafe, so always + # test your augmentation on masks + MASK_AUGMENTERS = ["Sequential", "SomeOf", "OneOf", "Sometimes", + "Fliplr", "Flipud", "CropAndPad", + "Affine", "PiecewiseAffine"] + + def hook(images, augmenter, parents, default): + """Determines which augmenters to apply to masks.""" + return augmenter.__class__.__name__ in MASK_AUGMENTERS + + # Store shapes before augmentation to compare + image_shape = image.shape + mask_shape = mask.shape + # Make augmenters deterministic to apply similarly to images and masks + det = augmentation.to_deterministic() + image = det.augment_image(image) + # Change mask to np.uint8 because imgaug doesn't support np.bool + mask = det.augment_image(mask.astype(np.uint8), + hooks=imgaug.HooksImages(activator=hook)) + # Verify that shapes didn't change + assert image.shape == image_shape, "Augmentation shouldn't change image size" + assert mask.shape == mask_shape, "Augmentation shouldn't change mask size" + # Change mask back to bool + mask = mask.astype(np.bool) + + # Note that some boxes might be all zeros if the corresponding mask got cropped out. + # and here is to filter them out + _idx = np.sum(mask, axis=(0, 1)) > 0 + mask = mask[:, :, _idx] + class_ids = class_ids[_idx] + # Bounding boxes. Note that some boxes might be all zeros + # if the corresponding mask got cropped out. + # bbox: [num_instances, (y1, x1, y2, x2)] + bbox = utils.extract_bboxes(mask) + + # Active classes + # Different datasets have different classes, so track the + # classes supported in the dataset of this image. + active_class_ids = np.zeros([dataset.num_classes], dtype=np.int32) + source_class_ids = dataset.source_class_ids[dataset.image_info[image_id]["source"]] + active_class_ids[source_class_ids] = 1 + + # Resize masks to smaller size to reduce memory usage + if use_mini_mask: + mask = utils.minimize_mask(bbox, mask, config.MINI_MASK_SHAPE) + + # Image meta data + image_meta = compose_image_meta(image_id, original_shape, image.shape, + window, scale, active_class_ids) + + return image, image_meta, class_ids, bbox, mask + + +def build_detection_targets(rpn_rois, gt_class_ids, gt_boxes, gt_masks, config): + """Generate targets for training Stage 2 classifier and mask heads. + This is not used in normal training. It's useful for debugging or to train + the Mask RCNN heads without using the RPN head. + + Inputs: + rpn_rois: [N, (y1, x1, y2, x2)] proposal boxes. + gt_class_ids: [instance count] Integer class IDs + gt_boxes: [instance count, (y1, x1, y2, x2)] + gt_masks: [height, width, instance count] Ground truth masks. Can be full + size or mini-masks. + + Returns: + rois: [TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)] + class_ids: [TRAIN_ROIS_PER_IMAGE]. Integer class IDs. + bboxes: [TRAIN_ROIS_PER_IMAGE, NUM_CLASSES, (y, x, log(h), log(w))]. Class-specific + bbox refinements. + masks: [TRAIN_ROIS_PER_IMAGE, height, width, NUM_CLASSES). Class specific masks cropped + to bbox boundaries and resized to neural network output size. + """ + assert rpn_rois.shape[0] > 0 + assert gt_class_ids.dtype == np.int32, "Expected int but got {}".format( + gt_class_ids.dtype) + assert gt_boxes.dtype == np.int32, "Expected int but got {}".format( + gt_boxes.dtype) + assert gt_masks.dtype == np.bool_, "Expected bool but got {}".format( + gt_masks.dtype) + + # It's common to add GT Boxes to ROIs but we don't do that here because + # according to XinLei Chen's paper, it doesn't help. + + # Trim empty padding in gt_boxes and gt_masks parts + instance_ids = np.where(gt_class_ids > 0)[0] + assert instance_ids.shape[0] > 0, "Image must contain instances." + gt_class_ids = gt_class_ids[instance_ids] + gt_boxes = gt_boxes[instance_ids] + gt_masks = gt_masks[:, :, instance_ids] + + # Compute areas of ROIs and ground truth boxes. + rpn_roi_area = (rpn_rois[:, 2] - rpn_rois[:, 0]) * \ + (rpn_rois[:, 3] - rpn_rois[:, 1]) + gt_box_area = (gt_boxes[:, 2] - gt_boxes[:, 0]) * \ + (gt_boxes[:, 3] - gt_boxes[:, 1]) + + # Compute overlaps [rpn_rois, gt_boxes] + overlaps = np.zeros((rpn_rois.shape[0], gt_boxes.shape[0])) + for i in range(overlaps.shape[1]): + gt = gt_boxes[i] + overlaps[:, i] = utils.compute_iou( + gt, rpn_rois, gt_box_area[i], rpn_roi_area) + + # Assign ROIs to GT boxes + rpn_roi_iou_argmax = np.argmax(overlaps, axis=1) + rpn_roi_iou_max = overlaps[np.arange( + overlaps.shape[0]), rpn_roi_iou_argmax] + # GT box assigned to each ROI + rpn_roi_gt_boxes = gt_boxes[rpn_roi_iou_argmax] + rpn_roi_gt_class_ids = gt_class_ids[rpn_roi_iou_argmax] + + # Positive ROIs are those with >= 0.5 IoU with a GT box. + fg_ids = np.where(rpn_roi_iou_max > 0.5)[0] + + # Negative ROIs are those with max IoU 0.1-0.5 (hard example mining) + # TODO: To hard example mine or not to hard example mine, that's the question + # bg_ids = np.where((rpn_roi_iou_max >= 0.1) & (rpn_roi_iou_max < 0.5))[0] + bg_ids = np.where(rpn_roi_iou_max < 0.5)[0] + + # Subsample ROIs. Aim for 33% foreground. + # FG + fg_roi_count = int(config.TRAIN_ROIS_PER_IMAGE * config.ROI_POSITIVE_RATIO) + if fg_ids.shape[0] > fg_roi_count: + keep_fg_ids = np.random.choice(fg_ids, fg_roi_count, replace=False) + else: + keep_fg_ids = fg_ids + # BG + remaining = config.TRAIN_ROIS_PER_IMAGE - keep_fg_ids.shape[0] + if bg_ids.shape[0] > remaining: + keep_bg_ids = np.random.choice(bg_ids, remaining, replace=False) + else: + keep_bg_ids = bg_ids + # Combine indices of ROIs to keep + keep = np.concatenate([keep_fg_ids, keep_bg_ids]) + # Need more? + remaining = config.TRAIN_ROIS_PER_IMAGE - keep.shape[0] + if remaining > 0: + # Looks like we don't have enough samples to maintain the desired + # balance. Reduce requirements and fill in the rest. This is + # likely different from the Mask RCNN paper. + + # There is a small chance we have neither fg nor bg samples. + if keep.shape[0] == 0: + # Pick bg regions with easier IoU threshold + bg_ids = np.where(rpn_roi_iou_max < 0.5)[0] + assert bg_ids.shape[0] >= remaining + keep_bg_ids = np.random.choice(bg_ids, remaining, replace=False) + assert keep_bg_ids.shape[0] == remaining + keep = np.concatenate([keep, keep_bg_ids]) + else: + # Fill the rest with repeated bg rois. + keep_extra_ids = np.random.choice( + keep_bg_ids, remaining, replace=True) + keep = np.concatenate([keep, keep_extra_ids]) + assert keep.shape[0] == config.TRAIN_ROIS_PER_IMAGE, \ + "keep doesn't match ROI batch size {}, {}".format( + keep.shape[0], config.TRAIN_ROIS_PER_IMAGE) + + # Reset the gt boxes assigned to BG ROIs. + rpn_roi_gt_boxes[keep_bg_ids, :] = 0 + rpn_roi_gt_class_ids[keep_bg_ids] = 0 + + # For each kept ROI, assign a class_id, and for FG ROIs also add bbox refinement. + rois = rpn_rois[keep] + roi_gt_boxes = rpn_roi_gt_boxes[keep] + roi_gt_class_ids = rpn_roi_gt_class_ids[keep] + roi_gt_assignment = rpn_roi_iou_argmax[keep] + + # Class-aware bbox deltas. [y, x, log(h), log(w)] + bboxes = np.zeros((config.TRAIN_ROIS_PER_IMAGE, + config.NUM_CLASSES, 4), dtype=np.float32) + pos_ids = np.where(roi_gt_class_ids > 0)[0] + bboxes[pos_ids, roi_gt_class_ids[pos_ids]] = utils.box_refinement( + rois[pos_ids], roi_gt_boxes[pos_ids, :4]) + # Normalize bbox refinements + bboxes /= config.BBOX_STD_DEV + + # Generate class-specific target masks + masks = np.zeros((config.TRAIN_ROIS_PER_IMAGE, config.MASK_SHAPE[0], config.MASK_SHAPE[1], config.NUM_CLASSES), + dtype=np.float32) + for i in pos_ids: + class_id = roi_gt_class_ids[i] + assert class_id > 0, "class id must be greater than 0" + gt_id = roi_gt_assignment[i] + class_mask = gt_masks[:, :, gt_id] + + if config.USE_MINI_MASK: + # Create a mask placeholder, the size of the image + placeholder = np.zeros(config.IMAGE_SHAPE[:2], dtype=bool) + # GT box + gt_y1, gt_x1, gt_y2, gt_x2 = gt_boxes[gt_id] + gt_w = gt_x2 - gt_x1 + gt_h = gt_y2 - gt_y1 + # Resize mini mask to size of GT box + placeholder[gt_y1:gt_y2, gt_x1:gt_x2] = \ + np.round(skimage.transform.resize( + class_mask, (gt_h, gt_w), order=1, mode="constant")).astype(bool) + # Place the mini batch in the placeholder + class_mask = placeholder + + # Pick part of the mask and resize it + y1, x1, y2, x2 = rois[i].astype(np.int32) + m = class_mask[y1:y2, x1:x2] + mask = skimage.transform.resize(m, config.MASK_SHAPE, order=1, mode="constant") + masks[i, :, :, class_id] = mask + + return rois, roi_gt_class_ids, bboxes, masks + + +def build_rpn_targets(image_shape, anchors, gt_class_ids, gt_boxes, config): + """Given the anchors and GT boxes, compute overlaps and identify positive + anchors and deltas to refine them to match their corresponding GT boxes. + + anchors: [num_anchors, (y1, x1, y2, x2)] + gt_class_ids: [num_gt_boxes] Integer class IDs. + gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] + + Returns: + rpn_match: [N] (int32) matches between anchors and GT boxes. + 1 = positive anchor, -1 = negative anchor, 0 = neutral + rpn_bbox: [N, (dy, dx, log(dh), log(dw))] Anchor bbox deltas. + """ + # RPN Match: 1 = positive anchor, -1 = negative anchor, 0 = neutral + rpn_match = np.zeros([anchors.shape[0]], dtype=np.int32) + # RPN bounding boxes: [max anchors per image, (dy, dx, log(dh), log(dw))] + rpn_bbox = np.zeros((config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4)) + + # Handle COCO crowds + # A crowd box in COCO is a bounding box around several instances. Exclude + # them from training. A crowd box is given a negative class ID. + crowd_ix = np.where(gt_class_ids < 0)[0] + if crowd_ix.shape[0] > 0: + # Filter out crowds from ground truth class IDs and boxes + non_crowd_ix = np.where(gt_class_ids > 0)[0] + crowd_boxes = gt_boxes[crowd_ix] + gt_class_ids = gt_class_ids[non_crowd_ix] + gt_boxes = gt_boxes[non_crowd_ix] + # Compute overlaps with crowd boxes [anchors, crowds] + crowd_overlaps = utils.compute_overlaps(anchors, crowd_boxes) + crowd_iou_max = np.amax(crowd_overlaps, axis=1) + no_crowd_bool = (crowd_iou_max < 0.001) + else: + # All anchors don't intersect a crowd + no_crowd_bool = np.ones([anchors.shape[0]], dtype=bool) + + # Compute overlaps [num_anchors, num_gt_boxes] + overlaps = utils.compute_overlaps(anchors, gt_boxes) + + # Match anchors to GT Boxes + # If an anchor overlaps a GT box with IoU >= 0.7 then it's positive. + # If an anchor overlaps a GT box with IoU < 0.3 then it's negative. + # Neutral anchors are those that don't match the conditions above, + # and they don't influence the loss function. + # However, don't keep any GT box unmatched (rare, but happens). Instead, + # match it to the closest anchor (even if its max IoU is < 0.3). + # + # 1. Set negative anchors first. They get overwritten below if a GT box is + # matched to them. Skip boxes in crowd areas. + anchor_iou_argmax = np.argmax(overlaps, axis=1) + anchor_iou_max = overlaps[np.arange(overlaps.shape[0]), anchor_iou_argmax] + rpn_match[(anchor_iou_max < 0.3) & (no_crowd_bool)] = -1 + # 2. Set an anchor for each GT box (regardless of IoU value). + # TODO: If multiple anchors have the same IoU match all of them + gt_iou_argmax = np.argmax(overlaps, axis=0) + rpn_match[gt_iou_argmax] = 1 + # 3. Set anchors with high overlap as positive. + rpn_match[anchor_iou_max >= 0.7] = 1 + + # Subsample to balance positive and negative anchors + # Don't let positives be more than half the anchors + ids = np.where(rpn_match == 1)[0] + extra = len(ids) - (config.RPN_TRAIN_ANCHORS_PER_IMAGE // 2) + if extra > 0: + # Reset the extra ones to neutral + ids = np.random.choice(ids, extra, replace=False) + rpn_match[ids] = 0 + # Same for negative proposals + ids = np.where(rpn_match == -1)[0] + extra = len(ids) - (config.RPN_TRAIN_ANCHORS_PER_IMAGE - + np.sum(rpn_match == 1)) + if extra > 0: + # Rest the extra ones to neutral + ids = np.random.choice(ids, extra, replace=False) + rpn_match[ids] = 0 + + # For positive anchors, compute shift and scale needed to transform them + # to match the corresponding GT boxes. + ids = np.where(rpn_match == 1)[0] + ix = 0 # index into rpn_bbox + # TODO: use box_refinement() rather than duplicating the code here + for i, a in zip(ids, anchors[ids]): + # Closest gt box (it might have IoU < 0.7) + gt = gt_boxes[anchor_iou_argmax[i]] + + # Convert coordinates to center plus width/height. + # GT Box + gt_h = gt[2] - gt[0] + gt_w = gt[3] - gt[1] + gt_center_y = gt[0] + 0.5 * gt_h + gt_center_x = gt[1] + 0.5 * gt_w + # Anchor + a_h = a[2] - a[0] + a_w = a[3] - a[1] + a_center_y = a[0] + 0.5 * a_h + a_center_x = a[1] + 0.5 * a_w + + # Compute the bbox refinement that the RPN should predict. + rpn_bbox[ix] = [ + (gt_center_y - a_center_y) / a_h, + (gt_center_x - a_center_x) / a_w, + np.log(gt_h / a_h), + np.log(gt_w / a_w), + ] + # Normalize + rpn_bbox[ix] /= config.RPN_BBOX_STD_DEV + ix += 1 + + return rpn_match, rpn_bbox + + +def generate_random_rois(image_shape, count, gt_class_ids, gt_boxes): + """Generates ROI proposals similar to what a region proposal network + would generate. + + image_shape: [Height, Width, Depth] + count: Number of ROIs to generate + gt_class_ids: [N] Integer ground truth class IDs + gt_boxes: [N, (y1, x1, y2, x2)] Ground truth boxes in pixels. + + Returns: [count, (y1, x1, y2, x2)] ROI boxes in pixels. + """ + # placeholder + rois = np.zeros((count, 4), dtype=np.int32) + + # Generate random ROIs around GT boxes (90% of count) + rois_per_box = int(0.9 * count / gt_boxes.shape[0]) + for i in range(gt_boxes.shape[0]): + gt_y1, gt_x1, gt_y2, gt_x2 = gt_boxes[i] + h = gt_y2 - gt_y1 + w = gt_x2 - gt_x1 + # random boundaries + r_y1 = max(gt_y1 - h, 0) + r_y2 = min(gt_y2 + h, image_shape[0]) + r_x1 = max(gt_x1 - w, 0) + r_x2 = min(gt_x2 + w, image_shape[1]) + + # To avoid generating boxes with zero area, we generate double what + # we need and filter out the extra. If we get fewer valid boxes + # than we need, we loop and try again. + while True: + y1y2 = np.random.randint(r_y1, r_y2, (rois_per_box * 2, 2)) + x1x2 = np.random.randint(r_x1, r_x2, (rois_per_box * 2, 2)) + # Filter out zero area boxes + threshold = 1 + y1y2 = y1y2[np.abs(y1y2[:, 0] - y1y2[:, 1]) >= + threshold][:rois_per_box] + x1x2 = x1x2[np.abs(x1x2[:, 0] - x1x2[:, 1]) >= + threshold][:rois_per_box] + if y1y2.shape[0] == rois_per_box and x1x2.shape[0] == rois_per_box: + break + + # Sort on axis 1 to ensure x1 <= x2 and y1 <= y2 and then reshape + # into x1, y1, x2, y2 order + x1, x2 = np.split(np.sort(x1x2, axis=1), 2, axis=1) + y1, y2 = np.split(np.sort(y1y2, axis=1), 2, axis=1) + box_rois = np.hstack([y1, x1, y2, x2]) + rois[rois_per_box * i:rois_per_box * (i + 1)] = box_rois + + # Generate random ROIs anywhere in the image (10% of count) + remaining_count = count - (rois_per_box * gt_boxes.shape[0]) + # To avoid generating boxes with zero area, we generate double what + # we need and filter out the extra. If we get fewer valid boxes + # than we need, we loop and try again. + while True: + y1y2 = np.random.randint(0, image_shape[0], (remaining_count * 2, 2)) + x1x2 = np.random.randint(0, image_shape[1], (remaining_count * 2, 2)) + # Filter out zero area boxes + threshold = 1 + y1y2 = y1y2[np.abs(y1y2[:, 0] - y1y2[:, 1]) >= + threshold][:remaining_count] + x1x2 = x1x2[np.abs(x1x2[:, 0] - x1x2[:, 1]) >= + threshold][:remaining_count] + if y1y2.shape[0] == remaining_count and x1x2.shape[0] == remaining_count: + break + + # Sort on axis 1 to ensure x1 <= x2 and y1 <= y2 and then reshape + # into x1, y1, x2, y2 order + x1, x2 = np.split(np.sort(x1x2, axis=1), 2, axis=1) + y1, y2 = np.split(np.sort(y1y2, axis=1), 2, axis=1) + global_rois = np.hstack([y1, x1, y2, x2]) + rois[-remaining_count:] = global_rois + return rois + + +def data_generator(dataset, config, shuffle=True, augment=False, augmentation=None, + random_rois=0, batch_size=1, detection_targets=False, + no_augmentation_sources=None): + """A generator that returns images and corresponding target class ids, + bounding box deltas, and masks. + + dataset: The Dataset object to pick data from + config: The model config object + shuffle: If True, shuffles the samples before every epoch + augment: (deprecated. Use augmentation instead). If true, apply random + image augmentation. Currently, only horizontal flipping is offered. + augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) augmentation. + For example, passing imgaug.augmenters.Fliplr(0.5) flips images + right/left 50% of the time. + random_rois: If > 0 then generate proposals to be used to train the + network classifier and mask heads. Useful if training + the Mask RCNN part without the RPN. + batch_size: How many images to return in each call + detection_targets: If True, generate detection targets (class IDs, bbox + deltas, and masks). Typically for debugging or visualizations because + in trainig detection targets are generated by DetectionTargetLayer. + no_augmentation_sources: Optional. List of sources to exclude for + augmentation. A source is string that identifies a dataset and is + defined in the Dataset class. + + Returns a Python generator. Upon calling next() on it, the + generator returns two lists, inputs and outputs. The contents + of the lists differs depending on the received arguments: + inputs list: + - images: [batch, H, W, C] + - image_meta: [batch, (meta data)] Image details. See compose_image_meta() + - rpn_match: [batch, N] Integer (1=positive anchor, -1=negative, 0=neutral) + - rpn_bbox: [batch, N, (dy, dx, log(dh), log(dw))] Anchor bbox deltas. + - gt_class_ids: [batch, MAX_GT_INSTANCES] Integer class IDs + - gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] + - gt_masks: [batch, height, width, MAX_GT_INSTANCES]. The height and width + are those of the image unless use_mini_mask is True, in which + case they are defined in MINI_MASK_SHAPE. + + outputs list: Usually empty in regular training. But if detection_targets + is True then the outputs list contains target class_ids, bbox deltas, + and masks. + """ + b = 0 # batch item index + image_index = -1 + image_ids = np.copy(dataset.image_ids) + error_count = 0 + no_augmentation_sources = no_augmentation_sources or [] + + # Anchors + # [anchor_count, (y1, x1, y2, x2)] + backbone_shapes = compute_backbone_shapes(config, config.IMAGE_SHAPE) + anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES, + config.RPN_ANCHOR_RATIOS, + backbone_shapes, + config.BACKBONE_STRIDES, + config.RPN_ANCHOR_STRIDE) + + # Keras requires a generator to run indefinitely. + while True: + try: + # Increment index to pick next image. Shuffle if at the start of an epoch. + image_index = (image_index + 1) % len(image_ids) + if shuffle and image_index == 0: + np.random.shuffle(image_ids) + + # Get GT bounding boxes and masks for image. + image_id = image_ids[image_index] + + # If the image source is not to be augmented pass None as augmentation + if dataset.image_info[image_id]['source'] in no_augmentation_sources: + image, image_meta, gt_class_ids, gt_boxes, gt_masks = \ + load_image_gt(dataset, config, image_id, augment=augment, + augmentation=None, + use_mini_mask=config.USE_MINI_MASK) + else: + image, image_meta, gt_class_ids, gt_boxes, gt_masks = \ + load_image_gt(dataset, config, image_id, augment=augment, + augmentation=augmentation, + use_mini_mask=config.USE_MINI_MASK) + + # Skip images that have no instances. This can happen in cases + # where we train on a subset of classes and the image doesn't + # have any of the classes we care about. + if not np.any(gt_class_ids > 0): + continue + + # RPN Targets + rpn_match, rpn_bbox = build_rpn_targets(image.shape, anchors, + gt_class_ids, gt_boxes, config) + + # Mask R-CNN Targets + if random_rois: + rpn_rois = generate_random_rois( + image.shape, random_rois, gt_class_ids, gt_boxes) + if detection_targets: + rois, mrcnn_class_ids, mrcnn_bbox, mrcnn_mask = \ + build_detection_targets( + rpn_rois, gt_class_ids, gt_boxes, gt_masks, config) + + # Init batch arrays + if b == 0: + batch_image_meta = np.zeros( + (batch_size,) + image_meta.shape, dtype=image_meta.dtype) + batch_rpn_match = np.zeros( + [batch_size, anchors.shape[0], 1], dtype=rpn_match.dtype) + batch_rpn_bbox = np.zeros( + [batch_size, config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4], dtype=rpn_bbox.dtype) + batch_images = np.zeros( + (batch_size,) + image.shape, dtype=np.float32) + batch_gt_class_ids = np.zeros( + (batch_size, config.MAX_GT_INSTANCES), dtype=np.int32) + batch_gt_boxes = np.zeros( + (batch_size, config.MAX_GT_INSTANCES, 4), dtype=np.int32) + batch_gt_masks = np.zeros( + (batch_size, gt_masks.shape[0], gt_masks.shape[1], + config.MAX_GT_INSTANCES), dtype=gt_masks.dtype) + if random_rois: + batch_rpn_rois = np.zeros( + (batch_size, rpn_rois.shape[0], 4), dtype=rpn_rois.dtype) + if detection_targets: + batch_rois = np.zeros( + (batch_size,) + rois.shape, dtype=rois.dtype) + batch_mrcnn_class_ids = np.zeros( + (batch_size,) + mrcnn_class_ids.shape, dtype=mrcnn_class_ids.dtype) + batch_mrcnn_bbox = np.zeros( + (batch_size,) + mrcnn_bbox.shape, dtype=mrcnn_bbox.dtype) + batch_mrcnn_mask = np.zeros( + (batch_size,) + mrcnn_mask.shape, dtype=mrcnn_mask.dtype) + + # If more instances than fits in the array, sub-sample from them. + if gt_boxes.shape[0] > config.MAX_GT_INSTANCES: + ids = np.random.choice( + np.arange(gt_boxes.shape[0]), config.MAX_GT_INSTANCES, replace=False) + gt_class_ids = gt_class_ids[ids] + gt_boxes = gt_boxes[ids] + gt_masks = gt_masks[:, :, ids] + + # Add to batch + batch_image_meta[b] = image_meta + batch_rpn_match[b] = rpn_match[:, np.newaxis] + batch_rpn_bbox[b] = rpn_bbox + batch_images[b] = mold_image(image.astype(np.float32), config) + batch_gt_class_ids[b, :gt_class_ids.shape[0]] = gt_class_ids + batch_gt_boxes[b, :gt_boxes.shape[0]] = gt_boxes + batch_gt_masks[b, :, :, :gt_masks.shape[-1]] = gt_masks + if random_rois: + batch_rpn_rois[b] = rpn_rois + if detection_targets: + batch_rois[b] = rois + batch_mrcnn_class_ids[b] = mrcnn_class_ids + batch_mrcnn_bbox[b] = mrcnn_bbox + batch_mrcnn_mask[b] = mrcnn_mask + b += 1 + + # Batch full? + if b >= batch_size: + inputs = [batch_images, batch_image_meta, batch_rpn_match, batch_rpn_bbox, + batch_gt_class_ids, batch_gt_boxes, batch_gt_masks] + outputs = [] + + if random_rois: + inputs.extend([batch_rpn_rois]) + if detection_targets: + inputs.extend([batch_rois]) + # Keras requires that output and targets have the same number of dimensions + batch_mrcnn_class_ids = np.expand_dims( + batch_mrcnn_class_ids, -1) + outputs.extend( + [batch_mrcnn_class_ids, batch_mrcnn_bbox, batch_mrcnn_mask]) + + yield inputs, outputs + + # start a new batch + b = 0 + except (GeneratorExit, KeyboardInterrupt): + raise + except: + # Log it and skip the image + logging.exception("Error processing image {}".format( + dataset.image_info[image_id])) + error_count += 1 + if error_count > 5: + raise + + +############################################################ +# MaskRCNN Class +############################################################ + +class MaskRCNN(): + """Encapsulates the Mask RCNN model functionality. + + The actual Keras model is in the keras_model property. + """ + + def __init__(self, mode, config, model_dir): + """ + mode: Either "training" or "inference" + config: A Sub-class of the Config class + model_dir: Directory to save training logs and trained weights + """ + assert mode in ['training', 'inference'] + self.mode = mode + self.config = config + self.model_dir = model_dir + self.set_log_dir() + self.keras_model = self.build(mode=mode, config=config) + + def build(self, mode, config): + """Build Mask R-CNN architecture. + input_shape: The shape of the input image. + mode: Either "training" or "inference". The inputs and + outputs of the model differ accordingly. + """ + assert mode in ['training', 'inference'] + + # Image size must be dividable by 2 multiple times + h, w = config.IMAGE_SHAPE[:2] + if h / 2 ** 6 != int(h / 2 ** 6) or w / 2 ** 6 != int(w / 2 ** 6): + raise Exception("Image size must be dividable by 2 at least 6 times " + "to avoid fractions when downscaling and upscaling." + "For example, use 256, 320, 384, 448, 512, ... etc. ") + + # Inputs + input_image = KL.Input( + shape=[None, None, 3], name="input_image") + input_image_meta = KL.Input(shape=[config.IMAGE_META_SIZE], + name="input_image_meta") + if mode == "training": + # RPN GT + input_rpn_match = KL.Input( + shape=[None, 1], name="input_rpn_match", dtype=tf.int32) + input_rpn_bbox = KL.Input( + shape=[None, 4], name="input_rpn_bbox", dtype=tf.float32) + + # Detection GT (class IDs, bounding boxes, and masks) + # 1. GT Class IDs (zero padded) + input_gt_class_ids = KL.Input( + shape=[None], name="input_gt_class_ids", dtype=tf.int32) + # 2. GT Boxes in pixels (zero padded) + # [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] in image coordinates + input_gt_boxes = KL.Input( + shape=[None, 4], name="input_gt_boxes", dtype=tf.float32) + # Normalize coordinates + gt_boxes = KL.Lambda(lambda x: norm_boxes_graph( + x, K.shape(input_image)[1:3]))(input_gt_boxes) + # 3. GT Masks (zero padded) + # [batch, height, width, MAX_GT_INSTANCES] + if config.USE_MINI_MASK: + input_gt_masks = KL.Input( + shape=[config.MINI_MASK_SHAPE[0], + config.MINI_MASK_SHAPE[1], None], + name="input_gt_masks", dtype=bool) + else: + input_gt_masks = KL.Input( + shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None], + name="input_gt_masks", dtype=bool) + elif mode == "inference": + # Anchors in normalized coordinates + input_anchors = KL.Input(shape=[None, 4], name="input_anchors") + + # Build the shared convolutional layers. + # Bottom-up Layers + # Returns a list of the last layers of each stage, 5 in total. + # Don't create the thead (stage 5), so we pick the 4th item in the list. + if callable(config.BACKBONE): + _, C2, C3, C4, C5 = config.BACKBONE(input_image, stage5=True, + train_bn=config.TRAIN_BN) + else: + _, C2, C3, C4, C5 = resnet_graph(input_image, config.BACKBONE, + stage5=True, train_bn=config.TRAIN_BN) + # Top-down Layers + # TODO: add assert to varify feature map sizes match what's in config + P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5) + P4 = KL.Add(name="fpn_p4add")([ + KL.UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5), + KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4)]) + P3 = KL.Add(name="fpn_p3add")([ + KL.UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4), + KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3)]) + P2 = KL.Add(name="fpn_p2add")([ + KL.UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3), + KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2)]) + # Attach 3x3 conv to all P layers to get the final feature maps. + P2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p2")(P2) + P3 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p3")(P3) + P4 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p4")(P4) + P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p5")(P5) + # P6 is used for the 5th anchor scale in RPN. Generated by + # subsampling from P5 with stride of 2. + P6 = KL.MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5) + + # Note that P6 is used in RPN, but not in the classifier heads. + rpn_feature_maps = [P2, P3, P4, P5, P6] + mrcnn_feature_maps = [P2, P3, P4, P5] + + # Anchors + if mode == "training": + anchors = self.get_anchors(config.IMAGE_SHAPE) + # Duplicate across the batch dimension because Keras requires it + # TODO: can this be optimized to avoid duplicating the anchors? + anchors = np.broadcast_to(anchors, (config.BATCH_SIZE,) + anchors.shape) + # A hack to get around Keras's bad support for constants + anchors = KL.Lambda(lambda x: tf.Variable(anchors), name="anchors")(input_image) + else: + anchors = input_anchors + + # RPN Model + rpn = build_rpn_model(config.RPN_ANCHOR_STRIDE, + len(config.RPN_ANCHOR_RATIOS), config.TOP_DOWN_PYRAMID_SIZE) + # Loop through pyramid layers + layer_outputs = [] # list of lists + for p in rpn_feature_maps: + layer_outputs.append(rpn([p])) + # Concatenate layer outputs + # Convert from list of lists of level outputs to list of lists + # of outputs across levels. + # e.g. [[a1, b1, c1], [a2, b2, c2]] => [[a1, a2], [b1, b2], [c1, c2]] + output_names = ["rpn_class_logits", "rpn_class", "rpn_bbox"] + outputs = list(zip(*layer_outputs)) + outputs = [KL.Concatenate(axis=1, name=n)(list(o)) + for o, n in zip(outputs, output_names)] + + rpn_class_logits, rpn_class, rpn_bbox = outputs + + # Generate proposals + # Proposals are [batch, N, (y1, x1, y2, x2)] in normalized coordinates + # and zero padded. + proposal_count = config.POST_NMS_ROIS_TRAINING if mode == "training" \ + else config.POST_NMS_ROIS_INFERENCE + rpn_rois = ProposalLayer( + proposal_count=proposal_count, + nms_threshold=config.RPN_NMS_THRESHOLD, + name="ROI", + config=config)([rpn_class, rpn_bbox, anchors]) + + if mode == "training": + # Class ID mask to mark class IDs supported by the dataset the image + # came from. + active_class_ids = KL.Lambda( + lambda x: parse_image_meta_graph(x)["active_class_ids"] + )(input_image_meta) + + if not config.USE_RPN_ROIS: + # Ignore predicted ROIs and use ROIs provided as an input. + input_rois = KL.Input(shape=[config.POST_NMS_ROIS_TRAINING, 4], + name="input_roi", dtype=np.int32) + # Normalize coordinates + target_rois = KL.Lambda(lambda x: norm_boxes_graph( + x, K.shape(input_image)[1:3]))(input_rois) + else: + target_rois = rpn_rois + + # Generate detection targets + # Subsamples proposals and generates target outputs for training + # Note that proposal class IDs, gt_boxes, and gt_masks are zero + # padded. Equally, returned rois and targets are zero padded. + rois, target_class_ids, target_bbox, target_mask = \ + DetectionTargetLayer(config, name="proposal_targets")([ + target_rois, input_gt_class_ids, gt_boxes, input_gt_masks]) + + # Network Heads + # TODO: verify that this handles zero padded ROIs + mrcnn_class_logits, mrcnn_class, mrcnn_bbox = \ + fpn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta, + config.POOL_SIZE, config.NUM_CLASSES, + train_bn=config.TRAIN_BN, + fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) + + mrcnn_mask = build_fpn_mask_graph(rois, mrcnn_feature_maps, + input_image_meta, + config.MASK_POOL_SIZE, + config.NUM_CLASSES, + train_bn=config.TRAIN_BN) + + # TODO: clean up (use tf.identify if necessary) + output_rois = KL.Lambda(lambda x: x * 1, name="output_rois")(rois) + + # Losses + rpn_class_loss = KL.Lambda(lambda x: rpn_class_loss_graph(*x), name="rpn_class_loss")( + [input_rpn_match, rpn_class_logits]) + rpn_bbox_loss = KL.Lambda(lambda x: rpn_bbox_loss_graph(config, *x), name="rpn_bbox_loss")( + [input_rpn_bbox, input_rpn_match, rpn_bbox]) + class_loss = KL.Lambda(lambda x: mrcnn_class_loss_graph(*x), name="mrcnn_class_loss")( + [target_class_ids, mrcnn_class_logits, active_class_ids]) + bbox_loss = KL.Lambda(lambda x: mrcnn_bbox_loss_graph(*x), name="mrcnn_bbox_loss")( + [target_bbox, target_class_ids, mrcnn_bbox]) + mask_loss = KL.Lambda(lambda x: mrcnn_mask_loss_graph(*x), name="mrcnn_mask_loss")( + [target_mask, target_class_ids, mrcnn_mask]) + + # Model + inputs = [input_image, input_image_meta, + input_rpn_match, input_rpn_bbox, input_gt_class_ids, input_gt_boxes, input_gt_masks] + if not config.USE_RPN_ROIS: + inputs.append(input_rois) + outputs = [rpn_class_logits, rpn_class, rpn_bbox, + mrcnn_class_logits, mrcnn_class, mrcnn_bbox, mrcnn_mask, + rpn_rois, output_rois, + rpn_class_loss, rpn_bbox_loss, class_loss, bbox_loss, mask_loss] + model = KM.Model(inputs, outputs, name='mask_rcnn') + else: + # Network Heads + # Proposal classifier and BBox regressor heads + mrcnn_class_logits, mrcnn_class, mrcnn_bbox = \ + fpn_classifier_graph(rpn_rois, mrcnn_feature_maps, input_image_meta, + config.POOL_SIZE, config.NUM_CLASSES, + train_bn=config.TRAIN_BN, + fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) + + # Detections + # output is [batch, num_detections, (y1, x1, y2, x2, class_id, score)] in + # normalized coordinates + detections = DetectionLayer(config, name="mrcnn_detection")( + [rpn_rois, mrcnn_class, mrcnn_bbox, input_image_meta]) + + # Create masks for detections + detection_boxes = KL.Lambda(lambda x: x[..., :4])(detections) + mrcnn_mask = build_fpn_mask_graph(detection_boxes, mrcnn_feature_maps, + input_image_meta, + config.MASK_POOL_SIZE, + config.NUM_CLASSES, + train_bn=config.TRAIN_BN) + + model = KM.Model([input_image, input_image_meta, input_anchors], + [detections, mrcnn_class, mrcnn_bbox, + mrcnn_mask, rpn_rois, rpn_class, rpn_bbox], + name='mask_rcnn') + + return model + + def find_last(self): + """Finds the last checkpoint file of the last trained model in the + model directory. + Returns: + The path of the last checkpoint file + """ + # Get directory names. Each directory corresponds to a model + dir_names = next(os.walk(self.model_dir))[1] + print("dirnames", dir_names) + key = self.config.NAME.lower() + print("key", key) + dir_names = filter(lambda f: f.startswith(key), dir_names) + dir_names = sorted(dir_names) + if not dir_names: + import errno + raise FileNotFoundError( + errno.ENOENT, + "Could not find model directory under {}".format(self.model_dir)) + # Pick last directory + dir_name = os.path.join(self.model_dir, dir_names[-1]) + # Find the last checkpoint + checkpoints = next(os.walk(dir_name))[2] + checkpoints = filter(lambda f: f.startswith("mask_rcnn"), checkpoints) + checkpoints = sorted(checkpoints) + if not checkpoints: + import errno + raise FileNotFoundError( + errno.ENOENT, "Could not find weight files in {}".format(dir_name)) + checkpoint = os.path.join(dir_name, checkpoints[-1]) + return checkpoint + + def load_weights(self, filepath, by_name=False, exclude=None): + """Modified version of the corresponding Keras function with + the addition of multi-GPU support and the ability to exclude + some layers from loading. + exclude: list of layer names to exclude + """ + import h5py + # Conditional import to support versions of Keras before 2.2 + # TODO: remove in about 6 months (end of 2018) + try: + from keras.engine import saving as saving + except ImportError: + # Keras before 2.2 used the 'topology' namespace. + from keras.engine import saving as saving + + if exclude: + by_name = True + + if h5py is None: + raise ImportError('`load_weights` requires h5py.') + f = h5py.File(filepath, mode='r') + if 'layer_names' not in f.attrs and 'model_weights' in f: + f = f['model_weights'] + + # In multi-GPU training, we wrap the model. Get layers + # of the inner model because they have the weights. + keras_model = self.keras_model + layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model") \ + else keras_model.layers + + # Exclude some layers + if exclude: + layers = filter(lambda l: l.name not in exclude, layers) + + if by_name: + saving.load_weights_from_hdf5_group_by_name(f, layers) + else: + saving.load_weights_from_hdf5_group(f, layers) + if hasattr(f, 'close'): + f.close() + + # Update the log directory + self.set_log_dir(filepath) + + def get_imagenet_weights(self): + """Downloads ImageNet trained weights from Keras. + Returns path to weights file. + """ + from keras.utils.data_utils import get_file + TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/' \ + 'releases/download/v0.2/' \ + 'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' + weights_path = get_file('resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5', + TF_WEIGHTS_PATH_NO_TOP, + cache_subdir='models', + md5_hash='a268eb855778b3df3c7506639542a6af') + return weights_path + + def compile(self, learning_rate, momentum): + """Gets the model ready for training. Adds losses, regularization, and + metrics. Then calls the Keras compile() function. + """ + # Optimizer object + optimizer = keras.optimizers.SGD( + lr=learning_rate, momentum=momentum, + clipnorm=self.config.GRADIENT_CLIP_NORM) + # Add Losses + # First, clear previously set losses to avoid duplication + self.keras_model._losses = [] + self.keras_model._per_input_losses = {} + loss_names = [ + "rpn_class_loss", "rpn_bbox_loss", + "mrcnn_class_loss", "mrcnn_bbox_loss", "mrcnn_mask_loss"] + for name in loss_names: + layer = self.keras_model.get_layer(name) + if layer.output in self.keras_model.losses: + continue + loss = ( + tf.reduce_mean(layer.output, keepdims=True) + * self.config.LOSS_WEIGHTS.get(name, 1.)) + self.keras_model.add_loss(loss) + + # Add L2 Regularization + # Skip gamma and beta weights of batch normalization layers. + reg_losses = [ + keras.regularizers.l2(self.config.WEIGHT_DECAY)(w) / tf.cast(tf.size(w), tf.float32) + for w in self.keras_model.trainable_weights + if 'gamma' not in w.name and 'beta' not in w.name] + self.keras_model.add_loss(tf.add_n(reg_losses)) + + # Compile + self.keras_model.compile( + optimizer=optimizer, + loss=[None] * len(self.keras_model.outputs)) + + # Add metrics for losses + for name in loss_names: + if name in self.keras_model.metrics_names: + continue + layer = self.keras_model.get_layer(name) + self.keras_model.metrics_names.append(name) + loss = ( + tf.reduce_mean(layer.output, keepdims=True) + * self.config.LOSS_WEIGHTS.get(name, 1.)) + self.keras_model.metrics_tensors.append(loss) + + def set_trainable(self, layer_regex, keras_model=None, indent=0, verbose=1): + """Sets model layers as trainable if their names match + the given regular expression. + """ + # Print message on the first call (but not on recursive calls) + if verbose > 0 and keras_model is None: + log("Selecting layers to train") + + keras_model = keras_model or self.keras_model + + # In multi-GPU training, we wrap the model. Get layers + # of the inner model because they have the weights. + layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model") \ + else keras_model.layers + + for layer in layers: + # Is the layer a model? + if layer.__class__.__name__ == 'Model': + print("In model: ", layer.name) + self.set_trainable( + layer_regex, keras_model=layer, indent=indent + 4) + continue + + if not layer.weights: + continue + # Is it trainable? + trainable = bool(re.fullmatch(layer_regex, layer.name)) + # Update layer. If layer is a container, update inner layer. + if layer.__class__.__name__ == 'TimeDistributed': + layer.layer.trainable = trainable + else: + layer.trainable = trainable + # Print trainable layer names + if trainable and verbose > 0: + log("{}{:20} ({})".format(" " * indent, layer.name, + layer.__class__.__name__)) + + def set_log_dir(self, model_path=None): + """Sets the model log directory and epoch counter. + + model_path: If None, or a format different from what this code uses + then set a new log directory and start epochs from 0. Otherwise, + extract the log directory and the epoch counter from the file + name. + """ + # Set date and epoch counter as if starting a new model + self.epoch = 0 + now = datetime.datetime.now() + + # If we have a model path with date and epochs use them + if model_path: + # Continue from we left of. Get epoch and date from the file name + # A sample model path might look like: + # /path/to/logs/coco20171029T2315/mask_rcnn_coco_0001.h5 + regex = r".*/[\w-]+(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})/mask\_rcnn\_[\w-]+(\d{4})\.h5" + m = re.match(regex, model_path) + if m: + now = datetime.datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), + int(m.group(4)), int(m.group(5))) + # Epoch number in file is 1-based, and in Keras code it's 0-based. + # So, adjust for that then increment by one to start from the next epoch + self.epoch = int(m.group(6)) - 1 + 1 + print('Re-starting from epoch %d' % self.epoch) + + # Directory for training logs + # self.log_dir = os.path.join(self.model_dir, "{}{:%Y%m%dT%H%M}".format( + # self.config.NAME.lower(), now)) + # + # # Create log_dir if not exists + # if not os.path.exists(self.log_dir): + # os.makedirs(self.log_dir) + + # Path to save after each epoch. Include placeholders that get filled by Keras. + # self.checkpoint_path = os.path.join(self.log_dir, "mask_rcnn_{}_*epoch*.h5".format( + # self.config.NAME.lower())) + # self.checkpoint_path = self.checkpoint_path.replace( + # "*epoch*", "{epoch:04d}") + + def train(self, train_dataset, val_dataset, learning_rate, epochs, layers, + augmentation=None, custom_callbacks=None, no_augmentation_sources=None): + """Train the model. + train_dataset, val_dataset: Training and validation Dataset objects. + learning_rate: The learning rate to train with + epochs: Number of training epochs. Note that previous training epochs + are considered to be done alreay, so this actually determines + the epochs to train in total rather than in this particaular + call. + layers: Allows selecting wich layers to train. It can be: + - A regular expression to match layer names to train + - One of these predefined values: + heads: The RPN, classifier and mask heads of the network + all: All the layers + 3+: Train Resnet stage 3 and up + 4+: Train Resnet stage 4 and up + 5+: Train Resnet stage 5 and up + augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) + augmentation. For example, passing imgaug.augmenters.Fliplr(0.5) + flips images right/left 50% of the time. You can pass complex + augmentations as well. This augmentation applies 50% of the + time, and when it does it flips images right/left half the time + and adds a Gaussian blur with a random sigma in range 0 to 5. + + augmentation = imgaug.augmenters.Sometimes(0.5, [ + imgaug.augmenters.Fliplr(0.5), + imgaug.augmenters.GaussianBlur(sigma=(0.0, 5.0)) + ]) + custom_callbacks: Optional. Add custom callbacks to be called + with the keras fit_generator method. Must be list of type keras.callbacks. + no_augmentation_sources: Optional. List of sources to exclude for + augmentation. A source is string that identifies a dataset and is + defined in the Dataset class. + """ + assert self.mode == "training", "Create model in training mode." + + # Pre-defined layer regular expressions + layer_regex = { + # all layers but the backbone + "heads": r"(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)", + # From a specific Resnet stage and up + "3+": r"(res3.*)|(bn3.*)|(res4.*)|(bn4.*)|(res5.*)|(bn5.*)|(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)", + "4+": r"(res4.*)|(bn4.*)|(res5.*)|(bn5.*)|(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)", + "5+": r"(res5.*)|(bn5.*)|(mrcnn\_.*)|(rpn\_.*)|(fpn\_.*)", + # All layers + "all": ".*", + } + if layers in layer_regex.keys(): + layers = layer_regex[layers] + + # Data generators + train_generator = data_generator(train_dataset, self.config, shuffle=True, + augmentation=augmentation, + batch_size=self.config.BATCH_SIZE, + no_augmentation_sources=no_augmentation_sources) + val_generator = data_generator(val_dataset, self.config, shuffle=True, + batch_size=self.config.BATCH_SIZE) + + # Callbacks + callbacks = [ + keras.callbacks.TensorBoard(log_dir=self.log_dir, + histogram_freq=0, write_graph=True, write_images=False), + keras.callbacks.ModelCheckpoint(self.checkpoint_path, + verbose=0, save_weights_only=True), + ] + + # Add custom callbacks to the list + if custom_callbacks: + callbacks += custom_callbacks + + # Train + log("\nStarting at epoch {}. LR={}\n".format(self.epoch, learning_rate)) + log("Checkpoint Path: {}".format(self.checkpoint_path)) + self.set_trainable(layers) + self.compile(learning_rate, self.config.LEARNING_MOMENTUM) + + # Work-around for Windows: Keras fails on Windows when using + # multiprocessing workers. See discussion here: + # https://github.com/matterport/Mask_RCNN/issues/13#issuecomment-353124009 + if os.name is 'nt': + workers = 0 + else: + workers = multiprocessing.cpu_count() + + self.keras_model.fit_generator( + train_generator, + initial_epoch=self.epoch, + epochs=epochs, + steps_per_epoch=self.config.STEPS_PER_EPOCH, + callbacks=callbacks, + validation_data=val_generator, + validation_steps=self.config.VALIDATION_STEPS, + max_queue_size=100, + workers=workers, + use_multiprocessing=True, + ) + self.epoch = max(self.epoch, epochs) + + def mold_inputs(self, images): + """Takes a list of images and modifies them to the format expected + as an input to the neural network. + images: List of image matrices [height,width,depth]. Images can have + different sizes. + + Returns 3 Numpy matrices: + molded_images: [N, h, w, 3]. Images resized and normalized. + image_metas: [N, length of meta data]. Details about each image. + windows: [N, (y1, x1, y2, x2)]. The portion of the image that has the + original image (padding excluded). + """ + molded_images = [] + image_metas = [] + windows = [] + for image in images: + # Resize image + # TODO: move resizing to mold_image() + molded_image, window, scale, padding, crop = utils.resize_image( + image, + min_dim=self.config.IMAGE_MIN_DIM, + min_scale=self.config.IMAGE_MIN_SCALE, + max_dim=self.config.IMAGE_MAX_DIM, + mode=self.config.IMAGE_RESIZE_MODE) + molded_image = mold_image(molded_image, self.config) + # Build image_meta + image_meta = compose_image_meta( + 0, image.shape, molded_image.shape, window, scale, + np.zeros([self.config.NUM_CLASSES], dtype=np.int32)) + # Append + molded_images.append(molded_image) + windows.append(window) + image_metas.append(image_meta) + # Pack into arrays + molded_images = np.stack(molded_images) + image_metas = np.stack(image_metas) + windows = np.stack(windows) + return molded_images, image_metas, windows + + def unmold_detections(self, detections, mrcnn_mask, original_image_shape, + image_shape, window): + """Reformats the detections of one image from the format of the neural + network output to a format suitable for use in the rest of the + application. + + detections: [N, (y1, x1, y2, x2, class_id, score)] in normalized coordinates + mrcnn_mask: [N, height, width, num_classes] + original_image_shape: [H, W, C] Original image shape before resizing + image_shape: [H, W, C] Shape of the image after resizing and padding + window: [y1, x1, y2, x2] Pixel coordinates of box in the image where the real + image is excluding the padding. + + Returns: + boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels + class_ids: [N] Integer class IDs for each bounding box + scores: [N] Float probability scores of the class_id + masks: [height, width, num_instances] Instance masks + """ + # How many detections do we have? + # Detections array is padded with zeros. Find the first class_id == 0. + zero_ix = np.where(detections[:, 4] == 0)[0] + N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0] + + # Extract boxes, class_ids, scores, and class-specific masks + boxes = detections[:N, :4] + class_ids = detections[:N, 4].astype(np.int32) + scores = detections[:N, 5] + masks = mrcnn_mask[np.arange(N), :, :, class_ids] + + # Translate normalized coordinates in the resized image to pixel + # coordinates in the original image before resizing + window = utils.norm_boxes(window, image_shape[:2]) + wy1, wx1, wy2, wx2 = window + shift = np.array([wy1, wx1, wy1, wx1]) + wh = wy2 - wy1 # window height + ww = wx2 - wx1 # window width + scale = np.array([wh, ww, wh, ww]) + # Convert boxes to normalized coordinates on the window + boxes = np.divide(boxes - shift, scale) + # Convert boxes to pixel coordinates on the original image + boxes = utils.denorm_boxes(boxes, original_image_shape[:2]) + + # Filter out detections with zero area. Happens in early training when + # network weights are still random + exclude_ix = np.where( + (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0] + if exclude_ix.shape[0] > 0: + boxes = np.delete(boxes, exclude_ix, axis=0) + class_ids = np.delete(class_ids, exclude_ix, axis=0) + scores = np.delete(scores, exclude_ix, axis=0) + masks = np.delete(masks, exclude_ix, axis=0) + N = class_ids.shape[0] + + # Resize masks to original image size and set boundary threshold. + full_masks = [] + for i in range(N): + # Convert neural network mask to full size mask + full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape) + full_masks.append(full_mask) + full_masks = np.stack(full_masks, axis=-1) \ + if full_masks else np.empty(original_image_shape[:2] + (0,)) + + return boxes, class_ids, scores, full_masks + + def detect(self, images, verbose=0): + """Runs the detection pipeline. + + images: List of images, potentially of different sizes. + + Returns a list of dicts, one dict per image. The dict contains: + rois: [N, (y1, x1, y2, x2)] detection bounding boxes + class_ids: [N] int class IDs + scores: [N] float probability scores for the class IDs + masks: [H, W, N] instance binary masks + """ + assert self.mode == "inference", "Create model in inference mode." + assert len( + images) == self.config.BATCH_SIZE, "len(images) must be equal to BATCH_SIZE" + + if verbose: + log("Processing {} images".format(len(images))) + for image in images: + log("image", image) + + # Mold inputs to format expected by the neural network + molded_images, image_metas, windows = self.mold_inputs(images) + + # Validate image sizes + # All images in a batch MUST be of the same size + image_shape = molded_images[0].shape + for g in molded_images[1:]: + assert g.shape == image_shape, \ + "After resizing, all images must have the same size. Check IMAGE_RESIZE_MODE and image sizes." + + # Anchors + print("$" * 50) + print('image shape:', image_shape) + anchors = self.get_anchors(image_shape) + print('achors shape:', anchors.shape) + # Duplicate across the batch dimension because Keras requires it + # TODO: can this be optimized to avoid duplicating the anchors? + anchors = np.broadcast_to(anchors, (self.config.BATCH_SIZE,) + anchors.shape) + print('achors shape:', anchors.shape) + print('Config batch size:', self.config.BATCH_SIZE) + print('META shape:', image_metas.shape) + print("*" * 70) + print(molded_images.shape) + print(image_metas.shape) + print(anchors.shape) + + if verbose: + log("molded_images", molded_images) + log("image_metas", image_metas) + log("anchors", anchors) + # Run object detection + detections, _, _, mrcnn_mask, _, _, _ = \ + self.keras_model.predict([molded_images, image_metas, anchors], verbose=0) + print("#" * 70) + print(detections) + print(mrcnn_mask) + # Process detections + results = [] + for i, image in enumerate(images): + final_rois, final_class_ids, final_scores, final_masks = \ + self.unmold_detections(detections[i], mrcnn_mask[i], + image.shape, molded_images[i].shape, + windows[i]) + results.append({ + "rois": final_rois, + "class_ids": final_class_ids, + "scores": final_scores, + "masks": final_masks, + }) + print(final_masks.shape) + return results + + def detect_molded(self, molded_images, image_metas, verbose=0): + """Runs the detection pipeline, but expect inputs that are + molded already. Used mostly for debugging and inspecting + the model. + + molded_images: List of images loaded using load_image_gt() + image_metas: image meta data, also returned by load_image_gt() + + Returns a list of dicts, one dict per image. The dict contains: + rois: [N, (y1, x1, y2, x2)] detection bounding boxes + class_ids: [N] int class IDs + scores: [N] float probability scores for the class IDs + masks: [H, W, N] instance binary masks + """ + assert self.mode == "inference", "Create model in inference mode." + assert len(molded_images) == self.config.BATCH_SIZE, \ + "Number of images must be equal to BATCH_SIZE" + + if verbose: + log("Processing {} images".format(len(molded_images))) + for image in molded_images: + log("image", image) + + # Validate image sizes + # All images in a batch MUST be of the same size + image_shape = molded_images[0].shape + for g in molded_images[1:]: + assert g.shape == image_shape, "Images must have the same size" + + # Anchors + anchors = self.get_anchors(image_shape) + # Duplicate across the batch dimension because Keras requires it + # TODO: can this be optimized to avoid duplicating the anchors? + anchors = np.broadcast_to(anchors, (self.config.BATCH_SIZE,) + anchors.shape) + + if verbose: + log("molded_images", molded_images) + log("image_metas", image_metas) + log("anchors", anchors) + # Run object detection + detections, _, _, mrcnn_mask, _, _, _ = \ + self.keras_model.predict([molded_images, image_metas, anchors], verbose=0) + # Process detections + results = [] + for i, image in enumerate(molded_images): + window = [0, 0, image.shape[0], image.shape[1]] + final_rois, final_class_ids, final_scores, final_masks = \ + self.unmold_detections(detections[i], mrcnn_mask[i], + image.shape, molded_images[i].shape, + window) + results.append({ + "rois": final_rois, + "class_ids": final_class_ids, + "scores": final_scores, + "masks": final_masks, + }) + return results + + def get_anchors(self, image_shape): + """Returns anchor pyramid for the given image size.""" + backbone_shapes = compute_backbone_shapes(self.config, image_shape) + # Cache anchors and reuse if image shape is the same + if not hasattr(self, "_anchor_cache"): + self._anchor_cache = {} + if not tuple(image_shape) in self._anchor_cache: + # Generate Anchors + a = utils.generate_pyramid_anchors( + self.config.RPN_ANCHOR_SCALES, + self.config.RPN_ANCHOR_RATIOS, + backbone_shapes, + self.config.BACKBONE_STRIDES, + self.config.RPN_ANCHOR_STRIDE) + # Keep a copy of the latest anchors in pixel coordinates because + # it's used in inspect_model notebooks. + # TODO: Remove this after the notebook are refactored to not use it + self.anchors = a + # Normalize coordinates + self._anchor_cache[tuple(image_shape)] = utils.norm_boxes(a, image_shape[:2]) + return self._anchor_cache[tuple(image_shape)] + + def ancestor(self, tensor, name, checked=None): + """Finds the ancestor of a TF tensor in the computation graph. + tensor: TensorFlow symbolic tensor. + name: Name of ancestor tensor to find + checked: For internal use. A list of tensors that were already + searched to avoid loops in traversing the graph. + """ + checked = checked if checked is not None else [] + # Put a limit on how deep we go to avoid very long loops + if len(checked) > 500: + return None + # Convert name to a regex and allow matching a number prefix + # because Keras adds them automatically + if isinstance(name, str): + name = re.compile(name.replace("/", r"(\_\d+)*/")) + + parents = tensor.op.inputs + for p in parents: + if p in checked: + continue + if bool(re.fullmatch(name, p.name)): + return p + checked.append(p) + a = self.ancestor(p, name, checked) + if a is not None: + return a + return None + + def find_trainable_layer(self, layer): + """If a layer is encapsulated by another layer, this function + digs through the encapsulation and returns the layer that holds + the weights. + """ + if layer.__class__.__name__ == 'TimeDistributed': + return self.find_trainable_layer(layer.layer) + return layer + + def get_trainable_layers(self): + """Returns a list of layers that have weights.""" + layers = [] + # Loop through all layers + for l in self.keras_model.layers: + # If layer is a wrapper, find inner trainable layer + l = self.find_trainable_layer(l) + # Include layer if it has weights + if l.get_weights(): + layers.append(l) + return layers + + def run_graph(self, images, outputs, image_metas=None): + """Runs a sub-set of the computation graph that computes the given + outputs. + + image_metas: If provided, the images are assumed to be already + molded (i.e. resized, padded, and normalized) + + outputs: List of tuples (name, tensor) to compute. The tensors are + symbolic TensorFlow tensors and the names are for easy tracking. + + Returns an ordered dict of results. Keys are the names received in the + input and values are Numpy arrays. + """ + model = self.keras_model + + # Organize desired outputs into an ordered dict + outputs = OrderedDict(outputs) + for o in outputs.values(): + assert o is not None + + # Build a Keras function to run parts of the computation graph + inputs = model.inputs + if model.uses_learning_phase and not isinstance(K.learning_phase(), int): + inputs += [K.learning_phase()] + kf = K.function(model.inputs, list(outputs.values())) + + # Prepare inputs + if image_metas is None: + molded_images, image_metas, _ = self.mold_inputs(images) + else: + molded_images = images + image_shape = molded_images[0].shape + # Anchors + anchors = self.get_anchors(image_shape) + # Duplicate across the batch dimension because Keras requires it + # TODO: can this be optimized to avoid duplicating the anchors? + anchors = np.broadcast_to(anchors, (self.config.BATCH_SIZE,) + anchors.shape) + model_in = [molded_images, image_metas, anchors] + + # Run inference + if model.uses_learning_phase and not isinstance(K.learning_phase(), int): + model_in.append(0.) + outputs_np = kf(model_in) + + # Pack the generated Numpy arrays into a a dict and log the results. + outputs_np = OrderedDict([(k, v) + for k, v in zip(outputs.keys(), outputs_np)]) + for k, v in outputs_np.items(): + log(k, v) + return outputs_np + + +############################################################ +# Data Formatting +############################################################ + +def compose_image_meta(image_id, original_image_shape, image_shape, + window, scale, active_class_ids): + """Takes attributes of an image and puts them in one 1D array. + + image_id: An int ID of the image. Useful for debugging. + original_image_shape: [H, W, C] before resizing or padding. + image_shape: [H, W, C] after resizing and padding + window: (y1, x1, y2, x2) in pixels. The area of the image where the real + image is (excluding the padding) + scale: The scaling factor applied to the original image (float32) + active_class_ids: List of class_ids available in the dataset from which + the image came. Useful if training on images from multiple datasets + where not all classes are present in all datasets. + """ + meta = np.array( + [image_id] + # size=1 + list(original_image_shape) + # size=3 + list(image_shape) + # size=3 + list(window) + # size=4 (y1, x1, y2, x2) in image cooredinates + [scale] + # size=1 + list(active_class_ids) # size=num_classes + ) + return meta + + +def parse_image_meta(meta): + """Parses an array that contains image attributes to its components. + See compose_image_meta() for more details. + + meta: [batch, meta length] where meta length depends on NUM_CLASSES + + Returns a dict of the parsed values. + """ + image_id = meta[:, 0] + original_image_shape = meta[:, 1:4] + image_shape = meta[:, 4:7] + window = meta[:, 7:11] # (y1, x1, y2, x2) window of image in in pixels + scale = meta[:, 11] + active_class_ids = meta[:, 12:] + return { + "image_id": image_id.astype(np.int32), + "original_image_shape": original_image_shape.astype(np.int32), + "image_shape": image_shape.astype(np.int32), + "window": window.astype(np.int32), + "scale": scale.astype(np.float32), + "active_class_ids": active_class_ids.astype(np.int32), + } + + +def parse_image_meta_graph(meta): + """Parses a tensor that contains image attributes to its components. + See compose_image_meta() for more details. + + meta: [batch, meta length] where meta length depends on NUM_CLASSES + + Returns a dict of the parsed tensors. + """ + image_id = meta[:, 0] + original_image_shape = meta[:, 1:4] + image_shape = meta[:, 4:7] + window = meta[:, 7:11] # (y1, x1, y2, x2) window of image in in pixels + scale = meta[:, 11] + active_class_ids = meta[:, 12:] + return { + "image_id": image_id, + "original_image_shape": original_image_shape, + "image_shape": image_shape, + "window": window, + "scale": scale, + "active_class_ids": active_class_ids, + } + + +def mold_image(images, config): + """Expects an RGB image (or array of images) and subtracts + the mean pixel and converts it to float. Expects image + colors in RGB order. + """ + return images.astype(np.float32) - config.MEAN_PIXEL + + +def unmold_image(normalized_images, config): + """Takes a image normalized with mold() and returns the original.""" + return (normalized_images + config.MEAN_PIXEL).astype(np.uint8) + + +############################################################ +# Miscellenous Graph Functions +############################################################ + +def trim_zeros_graph(boxes, name=None): + """Often boxes are represented with matrices of shape [N, 4] and + are padded with zeros. This removes zero boxes. + + boxes: [N, 4] matrix of boxes. + non_zeros: [N] a 1D boolean mask identifying the rows to keep + """ + non_zeros = tf.cast(tf.reduce_sum(tf.abs(boxes), axis=1), tf.bool) + boxes = tf.boolean_mask(boxes, non_zeros, name=name) + return boxes, non_zeros + + +def batch_pack_graph(x, counts, num_rows): + """Picks different number of values from each row + in x depending on the values in counts. + """ + outputs = [] + for i in range(num_rows): + outputs.append(x[i, :counts[i]]) + return tf.concat(outputs, axis=0) + + +def norm_boxes_graph(boxes, shape): + """Converts boxes from pixel coordinates to normalized coordinates. + boxes: [..., (y1, x1, y2, x2)] in pixel coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [..., (y1, x1, y2, x2)] in normalized coordinates + """ + h, w = tf.split(tf.cast(shape, tf.float32), 2) + scale = tf.concat([h, w, h, w], axis=-1) - tf.constant(1.0) + shift = tf.constant([0., 0., 1., 1.]) + return tf.divide(boxes - shift, scale) + + +def denorm_boxes_graph(boxes, shape): + """Converts boxes from normalized coordinates to pixel coordinates. + boxes: [..., (y1, x1, y2, x2)] in normalized coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [..., (y1, x1, y2, x2)] in pixel coordinates + """ + h, w = tf.split(tf.cast(shape, tf.float32), 2) + scale = tf.concat([h, w, h, w], axis=-1) - tf.constant(1.0) + shift = tf.constant([0., 0., 1., 1.]) + return tf.cast(tf.round(tf.multiply(boxes, scale) + shift), tf.int32) diff --git a/mask_rcnn/tf_servable/user_config.py b/mask_rcnn/tf_servable/user_config.py new file mode 100644 index 00000000..ff4b22a6 --- /dev/null +++ b/mask_rcnn/tf_servable/user_config.py @@ -0,0 +1,24 @@ +import os +# User Define parameters + +# Make it True if you want to use the provided coco weights +is_coco = False + +# keras model file path +H5_WEIGHT_PATH = '../models/mask_rcnn_final.h5' +MODEL_DIR = os.path.dirname(H5_WEIGHT_PATH) + +# Path where the Frozen PB will be save +PATH_TO_SAVE_FROZEN_PB = 'frozen_model/' + +# Name for the Frozen PB name +FROZEN_NAME = 'mask_frozen_graph.pb' + +# PATH where to save serving model +PATH_TO_SAVE_TENSORFLOW_SERVING_MODEL = '../models/serving_model/' + +# Version of the serving model +VERSION_NUMBER = 1 + +# Number of classes that you have trained your model +NUMBER_OF_CLASSES = 6 diff --git a/mask_rcnn/tf_servable/utils.py b/mask_rcnn/tf_servable/utils.py new file mode 100644 index 00000000..026edf70 --- /dev/null +++ b/mask_rcnn/tf_servable/utils.py @@ -0,0 +1,884 @@ +import sys +import os +import math +import random +import cv2 +import numpy as np +import tensorflow as tf +import scipy +import skimage.color +# import skimage.io +import skimage.transform +import urllib.request +import shutil +import warnings + +# URL from which to download the latest COCO trained weights +COCO_MODEL_URL = "https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5" + + +############################################################ +# Bounding Boxes +############################################################ + +def extract_bboxes(mask): + """Compute bounding boxes from masks. + mask: [height, width, num_instances]. Mask pixels are either 1 or 0. + + Returns: bbox array [num_instances, (y1, x1, y2, x2)]. + """ + boxes = np.zeros([mask.shape[-1], 4], dtype=np.int32) + for i in range(mask.shape[-1]): + m = mask[:, :, i] + # Bounding box. + horizontal_indicies = np.where(np.any(m, axis=0))[0] + vertical_indicies = np.where(np.any(m, axis=1))[0] + if horizontal_indicies.shape[0]: + x1, x2 = horizontal_indicies[[0, -1]] + y1, y2 = vertical_indicies[[0, -1]] + # x2 and y2 should not be part of the box. Increment by 1. + x2 += 1 + y2 += 1 + else: + # No mask for this instance. Might happen due to + # resizing or cropping. Set bbox to zeros + x1, x2, y1, y2 = 0, 0, 0, 0 + boxes[i] = np.array([y1, x1, y2, x2]) + return boxes.astype(np.int32) + + +def compute_iou(box, boxes, box_area, boxes_area): + """Calculates IoU of the given box with the array of the given boxes. + box: 1D vector [y1, x1, y2, x2] + boxes: [boxes_count, (y1, x1, y2, x2)] + box_area: float. the area of 'box' + boxes_area: array of length boxes_count. + + Note: the areas are passed in rather than calculated here for + efficiency. Calculate once in the caller to avoid duplicate work. + """ + # Calculate intersection areas + y1 = np.maximum(box[0], boxes[:, 0]) + y2 = np.minimum(box[2], boxes[:, 2]) + x1 = np.maximum(box[1], boxes[:, 1]) + x2 = np.minimum(box[3], boxes[:, 3]) + intersection = np.maximum(x2 - x1, 0) * np.maximum(y2 - y1, 0) + union = box_area + boxes_area[:] - intersection[:] + iou = intersection / union + return iou + + +def compute_overlaps(boxes1, boxes2): + """Computes IoU overlaps between two sets of boxes. + boxes1, boxes2: [N, (y1, x1, y2, x2)]. + + For better performance, pass the largest set first and the smaller second. + """ + # Areas of anchors and GT boxes + area1 = (boxes1[:, 2] - boxes1[:, 0]) * (boxes1[:, 3] - boxes1[:, 1]) + area2 = (boxes2[:, 2] - boxes2[:, 0]) * (boxes2[:, 3] - boxes2[:, 1]) + + # Compute overlaps to generate matrix [boxes1 count, boxes2 count] + # Each cell contains the IoU value. + overlaps = np.zeros((boxes1.shape[0], boxes2.shape[0])) + for i in range(overlaps.shape[1]): + box2 = boxes2[i] + overlaps[:, i] = compute_iou(box2, boxes1, area2[i], area1) + return overlaps + + +def compute_overlaps_masks(masks1, masks2): + """Computes IoU overlaps between two sets of masks. + masks1, masks2: [Height, Width, instances] + """ + + # If either set of masks is empty return empty result + if masks1.shape[0] == 0 or masks2.shape[0] == 0: + return np.zeros((masks1.shape[0], masks2.shape[-1])) + # flatten masks and compute their areas + masks1 = np.reshape(masks1 > .5, (-1, masks1.shape[-1])).astype(np.float32) + masks2 = np.reshape(masks2 > .5, (-1, masks2.shape[-1])).astype(np.float32) + area1 = np.sum(masks1, axis=0) + area2 = np.sum(masks2, axis=0) + + # intersections and union + intersections = np.dot(masks1.T, masks2) + union = area1[:, None] + area2[None, :] - intersections + overlaps = intersections / union + + return overlaps + + +def non_max_suppression(boxes, scores, threshold): + """Performs non-maximum suppression and returns indices of kept boxes. + boxes: [N, (y1, x1, y2, x2)]. Notice that (y2, x2) lays outside the box. + scores: 1-D array of box scores. + threshold: Float. IoU threshold to use for filtering. + """ + assert boxes.shape[0] > 0 + if boxes.dtype.kind != "f": + boxes = boxes.astype(np.float32) + + # Compute box areas + y1 = boxes[:, 0] + x1 = boxes[:, 1] + y2 = boxes[:, 2] + x2 = boxes[:, 3] + area = (y2 - y1) * (x2 - x1) + + # Get indicies of boxes sorted by scores (highest first) + ixs = scores.argsort()[::-1] + + pick = [] + while len(ixs) > 0: + # Pick top box and add its index to the list + i = ixs[0] + pick.append(i) + # Compute IoU of the picked box with the rest + iou = compute_iou(boxes[i], boxes[ixs[1:]], area[i], area[ixs[1:]]) + # Identify boxes with IoU over the threshold. This + # returns indices into ixs[1:], so add 1 to get + # indices into ixs. + remove_ixs = np.where(iou > threshold)[0] + 1 + # Remove indices of the picked and overlapped boxes. + ixs = np.delete(ixs, remove_ixs) + ixs = np.delete(ixs, 0) + return np.array(pick, dtype=np.int32) + + +def apply_box_deltas(boxes, deltas): + """Applies the given deltas to the given boxes. + boxes: [N, (y1, x1, y2, x2)]. Note that (y2, x2) is outside the box. + deltas: [N, (dy, dx, log(dh), log(dw))] + """ + boxes = boxes.astype(np.float32) + # Convert to y, x, h, w + height = boxes[:, 2] - boxes[:, 0] + width = boxes[:, 3] - boxes[:, 1] + center_y = boxes[:, 0] + 0.5 * height + center_x = boxes[:, 1] + 0.5 * width + # Apply deltas + center_y += deltas[:, 0] * height + center_x += deltas[:, 1] * width + height *= np.exp(deltas[:, 2]) + width *= np.exp(deltas[:, 3]) + # Convert back to y1, x1, y2, x2 + y1 = center_y - 0.5 * height + x1 = center_x - 0.5 * width + y2 = y1 + height + x2 = x1 + width + return np.stack([y1, x1, y2, x2], axis=1) + + +def box_refinement_graph(box, gt_box): + """Compute refinement needed to transform box to gt_box. + box and gt_box are [N, (y1, x1, y2, x2)] + """ + box = tf.cast(box, tf.float32) + gt_box = tf.cast(gt_box, tf.float32) + + height = box[:, 2] - box[:, 0] + width = box[:, 3] - box[:, 1] + center_y = box[:, 0] + 0.5 * height + center_x = box[:, 1] + 0.5 * width + + gt_height = gt_box[:, 2] - gt_box[:, 0] + gt_width = gt_box[:, 3] - gt_box[:, 1] + gt_center_y = gt_box[:, 0] + 0.5 * gt_height + gt_center_x = gt_box[:, 1] + 0.5 * gt_width + + dy = (gt_center_y - center_y) / height + dx = (gt_center_x - center_x) / width + dh = tf.log(gt_height / height) + dw = tf.log(gt_width / width) + + result = tf.stack([dy, dx, dh, dw], axis=1) + return result + + +def box_refinement(box, gt_box): + """Compute refinement needed to transform box to gt_box. + box and gt_box are [N, (y1, x1, y2, x2)]. (y2, x2) is + assumed to be outside the box. + """ + box = box.astype(np.float32) + gt_box = gt_box.astype(np.float32) + + height = box[:, 2] - box[:, 0] + width = box[:, 3] - box[:, 1] + center_y = box[:, 0] + 0.5 * height + center_x = box[:, 1] + 0.5 * width + + gt_height = gt_box[:, 2] - gt_box[:, 0] + gt_width = gt_box[:, 3] - gt_box[:, 1] + gt_center_y = gt_box[:, 0] + 0.5 * gt_height + gt_center_x = gt_box[:, 1] + 0.5 * gt_width + + dy = (gt_center_y - center_y) / height + dx = (gt_center_x - center_x) / width + dh = np.log(gt_height / height) + dw = np.log(gt_width / width) + + return np.stack([dy, dx, dh, dw], axis=1) + + +############################################################ +# Dataset +############################################################ + +class Dataset(object): + """The base class for dataset classes. + To use it, create a new class that adds functions specific to the dataset + you want to use. For example: + + class CatsAndDogsDataset(Dataset): + def load_cats_and_dogs(self): + ... + def load_mask(self, image_id): + ... + def image_reference(self, image_id): + ... + + See COCODataset and ShapesDataset as examples. + """ + + def __init__(self, class_map=None): + self._image_ids = [] + self.image_info = [] + # Background is always the first class + self.class_info = [{"source": "", "id": 0, "name": "BG"}] + self.source_class_ids = {} + + def add_class(self, source, class_id, class_name): + assert "." not in source, "Source name cannot contain a dot" + # Does the class exist already? + for info in self.class_info: + if info['source'] == source and info["id"] == class_id: + # source.class_id combination already available, skip + return + # Add the class + self.class_info.append({ + "source": source, + "id": class_id, + "name": class_name, + }) + + def add_image(self, source, image_id, path, **kwargs): + image_info = { + "id": image_id, + "source": source, + "path": path, + } + image_info.update(kwargs) + self.image_info.append(image_info) + + def image_reference(self, image_id): + """Return a link to the image in its source Website or details about + the image that help looking it up or debugging it. + + Override for your dataset, but pass to this function + if you encounter images not in your dataset. + """ + return "" + + def prepare(self, class_map=None): + """Prepares the Dataset class for use. + + TODO: class map is not supported yet. When done, it should handle mapping + classes from different datasets to the same class ID. + """ + + def clean_name(name): + """Returns a shorter version of object names for cleaner display.""" + return ",".join(name.split(",")[:1]) + + # Build (or rebuild) everything else from the info dicts. + self.num_classes = len(self.class_info) + self.class_ids = np.arange(self.num_classes) + self.class_names = [clean_name(c["name"]) for c in self.class_info] + self.num_images = len(self.image_info) + self._image_ids = np.arange(self.num_images) + + # Mapping from source class and image IDs to internal IDs + self.class_from_source_map = {"{}.{}".format(info['source'], info['id']): id + for info, id in zip(self.class_info, self.class_ids)} + self.image_from_source_map = {"{}.{}".format(info['source'], info['id']): id + for info, id in zip(self.image_info, self.image_ids)} + + # Map sources to class_ids they support + self.sources = list(set([i['source'] for i in self.class_info])) + self.source_class_ids = {} + # Loop over datasets + for source in self.sources: + self.source_class_ids[source] = [] + # Find classes that belong to this dataset + for i, info in enumerate(self.class_info): + # Include BG class in all datasets + if i == 0 or source == info['source']: + self.source_class_ids[source].append(i) + + def map_source_class_id(self, source_class_id): + """Takes a source class ID and returns the int class ID assigned to it. + + For example: + dataset.map_source_class_id("coco.12") -> 23 + """ + return self.class_from_source_map[source_class_id] + + def get_source_class_id(self, class_id, source): + """Map an internal class ID to the corresponding class ID in the source dataset.""" + info = self.class_info[class_id] + assert info['source'] == source + return info['id'] + + def append_data(self, class_info, image_info): + self.external_to_class_id = {} + for i, c in enumerate(self.class_info): + for ds, id in c["map"]: + self.external_to_class_id[ds + str(id)] = i + + # Map external image IDs to internal ones. + self.external_to_image_id = {} + for i, info in enumerate(self.image_info): + self.external_to_image_id[info["ds"] + str(info["id"])] = i + + @property + def image_ids(self): + return self._image_ids + + def source_image_link(self, image_id): + """Returns the path or URL to the image. + Override this to return a URL to the image if it's available online for easy + debugging. + """ + return self.image_info[image_id]["path"] + + def load_image(self, image_id): + """Load the specified image and return a [H,W,3] Numpy array. + """ + # Load image + image = cv2.imread(self.image_info[image_id]['path']) + # image = skimage.io.imread(self.image_info[image_id]['path']) + # If grayscale. Convert to RGB for consistency. + if image.ndim != 3: + image = skimage.color.gray2rgb(image) + # If has an alpha channel, remove it for consistency + if image.shape[-1] == 4: + image = image[..., :3] + return image + + def load_mask(self, image_id): + """Load instance masks for the given image. + + Different datasets use different ways to store masks. Override this + method to load instance masks and return them in the form of am + array of binary masks of shape [height, width, instances]. + + Returns: + masks: A bool array of shape [height, width, instance count] with + a binary mask per instance. + class_ids: a 1D array of class IDs of the instance masks. + """ + # Override this function to load a mask from your dataset. + # Otherwise, it returns an empty mask. + mask = np.empty([0, 0, 0]) + class_ids = np.empty([0], np.int32) + return mask, class_ids + + +def resize_image(image, min_dim=None, max_dim=None, min_scale=None, mode="square"): + """Resizes an image keeping the aspect ratio unchanged. + + min_dim: if provided, resizes the image such that it's smaller + dimension == min_dim + max_dim: if provided, ensures that the image longest side doesn't + exceed this value. + min_scale: if provided, ensure that the image is scaled up by at least + this percent even if min_dim doesn't require it. + mode: Resizing mode. + none: No resizing. Return the image unchanged. + square: Resize and pad with zeros to get a square image + of size [max_dim, max_dim]. + pad64: Pads width and height with zeros to make them multiples of 64. + If min_dim or min_scale are provided, it scales the image up + before padding. max_dim is ignored in this mode. + The multiple of 64 is needed to ensure smooth scaling of feature + maps up and down the 6 levels of the FPN pyramid (2**6=64). + crop: Picks random crops from the image. First, scales the image based + on min_dim and min_scale, then picks a random crop of + size min_dim x min_dim. Can be used in training only. + max_dim is not used in this mode. + + Returns: + image: the resized image + window: (y1, x1, y2, x2). If max_dim is provided, padding might + be inserted in the returned image. If so, this window is the + coordinates of the image part of the full image (excluding + the padding). The x2, y2 pixels are not included. + scale: The scale factor used to resize the image + padding: Padding added to the image [(top, bottom), (left, right), (0, 0)] + """ + # Keep track of image dtype and return results in the same dtype + image_dtype = image.dtype + # Default window (y1, x1, y2, x2) and default scale == 1. + h, w = image.shape[:2] + window = (0, 0, h, w) + scale = 1 + padding = [(0, 0), (0, 0), (0, 0)] + crop = None + + if mode == "none": + return image, window, scale, padding, crop + + # Scale? + if min_dim: + # Scale up but not down + scale = max(1, min_dim / min(h, w)) + if min_scale and scale < min_scale: + scale = min_scale + + # Does it exceed max dim? + if max_dim and mode == "square": + image_max = max(h, w) + if round(image_max * scale) > max_dim: + scale = max_dim / image_max + + # Resize image using bilinear interpolation + if scale != 1: + image = skimage.transform.resize( + image, (round(h * scale), round(w * scale)), + order=1, mode="constant", preserve_range=True) + + # Need padding or cropping? + if mode == "square": + # Get new height and width + h, w = image.shape[:2] + top_pad = (max_dim - h) // 2 + bottom_pad = max_dim - h - top_pad + left_pad = (max_dim - w) // 2 + right_pad = max_dim - w - left_pad + padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)] + image = np.pad(image, padding, mode='constant', constant_values=0) + window = (top_pad, left_pad, h + top_pad, w + left_pad) + elif mode == "pad64": + h, w = image.shape[:2] + # Both sides must be divisible by 64 + assert min_dim % 64 == 0, "Minimum dimension must be a multiple of 64" + # Height + if h % 64 > 0: + max_h = h - (h % 64) + 64 + top_pad = (max_h - h) // 2 + bottom_pad = max_h - h - top_pad + else: + top_pad = bottom_pad = 0 + # Width + if w % 64 > 0: + max_w = w - (w % 64) + 64 + left_pad = (max_w - w) // 2 + right_pad = max_w - w - left_pad + else: + left_pad = right_pad = 0 + padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)] + image = np.pad(image, padding, mode='constant', constant_values=0) + window = (top_pad, left_pad, h + top_pad, w + left_pad) + elif mode == "crop": + # Pick a random crop + h, w = image.shape[:2] + y = random.randint(0, (h - min_dim)) + x = random.randint(0, (w - min_dim)) + crop = (y, x, min_dim, min_dim) + image = image[y:y + min_dim, x:x + min_dim] + window = (0, 0, min_dim, min_dim) + else: + raise Exception("Mode {} not supported".format(mode)) + return image.astype(image_dtype), window, scale, padding, crop + + +def resize_mask(mask, scale, padding, crop=None): + """Resizes a mask using the given scale and padding. + Typically, you get the scale and padding from resize_image() to + ensure both, the image and the mask, are resized consistently. + + scale: mask scaling factor + padding: Padding to add to the mask in the form + [(top, bottom), (left, right), (0, 0)] + """ + # Suppress warning from scipy 0.13.0, the output shape of zoom() is + # calculated with round() instead of int() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mask = scipy.ndimage.zoom(mask, zoom=[scale, scale, 1], order=0) + if crop is not None: + y, x, h, w = crop + mask = mask[y:y + h, x:x + w] + else: + mask = np.pad(mask, padding, mode='constant', constant_values=0) + return mask + + +def minimize_mask(bbox, mask, mini_shape): + """Resize masks to a smaller version to reduce memory load. + Mini-masks can be resized back to image scale using expand_masks() + + See inspect_data.ipynb notebook for more details. + """ + mini_mask = np.zeros(mini_shape + (mask.shape[-1],), dtype=bool) + for i in range(mask.shape[-1]): + # Pick slice and cast to bool in case load_mask() returned wrong dtype + m = mask[:, :, i].astype(bool) + y1, x1, y2, x2 = bbox[i][:4] + m = m[y1:y2, x1:x2] + if m.size == 0: + raise Exception("Invalid bounding box with area of zero") + # Resize with bilinear interpolation + m = skimage.transform.resize(m, mini_shape, order=1, mode="constant") + mini_mask[:, :, i] = np.around(m).astype(np.bool) + return mini_mask + + +def expand_mask(bbox, mini_mask, image_shape): + """Resizes mini masks back to image size. Reverses the change + of minimize_mask(). + + See inspect_data.ipynb notebook for more details. + """ + mask = np.zeros(image_shape[:2] + (mini_mask.shape[-1],), dtype=bool) + for i in range(mask.shape[-1]): + m = mini_mask[:, :, i] + y1, x1, y2, x2 = bbox[i][:4] + h = y2 - y1 + w = x2 - x1 + # Resize with bilinear interpolation + m = skimage.transform.resize(m, (h, w), order=1, mode="constant") + mask[y1:y2, x1:x2, i] = np.around(m).astype(np.bool) + return mask + + +# TODO: Build and use this function to reduce code duplication + + +def unmold_mask(mask, bbox, image_shape): + """Converts a mask generated by the neural network to a format similar + to its original shape. + mask: [height, width] of type float. A small, typically 28x28 mask. + bbox: [y1, x1, y2, x2]. The box to fit the mask in. + + Returns a binary mask with the same size as the original image. + """ + threshold = 0.5 + y1, x1, y2, x2 = bbox + mask = skimage.transform.resize(mask, (y2 - y1, x2 - x1), order=1, mode="constant") + mask = np.where(mask >= threshold, 1, 0).astype(np.bool) + + # Put the mask in the right location. + full_mask = np.zeros(image_shape[:2], dtype=np.bool) + full_mask[y1:y2, x1:x2] = mask + return full_mask + + +############################################################ +# Anchors +############################################################ + +def generate_anchors(scales, ratios, shape, feature_stride, anchor_stride): + """ + scales: 1D array of anchor sizes in pixels. Example: [32, 64, 128] + ratios: 1D array of anchor ratios of width/height. Example: [0.5, 1, 2] + shape: [height, width] spatial shape of the feature map over which + to generate anchors. + feature_stride: Stride of the feature map relative to the image in pixels. + anchor_stride: Stride of anchors on the feature map. For example, if the + value is 2 then generate anchors for every other feature map pixel. + """ + # Get all combinations of scales and ratios + scales, ratios = np.meshgrid(np.array(scales), np.array(ratios)) + scales = scales.flatten() + ratios = ratios.flatten() + + # Enumerate heights and widths from scales and ratios + heights = scales / np.sqrt(ratios) + widths = scales * np.sqrt(ratios) + + # Enumerate shifts in feature space + shifts_y = np.arange(0, shape[0], anchor_stride) * feature_stride + shifts_x = np.arange(0, shape[1], anchor_stride) * feature_stride + shifts_x, shifts_y = np.meshgrid(shifts_x, shifts_y) + + # Enumerate combinations of shifts, widths, and heights + box_widths, box_centers_x = np.meshgrid(widths, shifts_x) + box_heights, box_centers_y = np.meshgrid(heights, shifts_y) + + # Reshape to get a list of (y, x) and a list of (h, w) + box_centers = np.stack( + [box_centers_y, box_centers_x], axis=2).reshape([-1, 2]) + box_sizes = np.stack([box_heights, box_widths], axis=2).reshape([-1, 2]) + + # Convert to corner coordinates (y1, x1, y2, x2) + boxes = np.concatenate([box_centers - 0.5 * box_sizes, + box_centers + 0.5 * box_sizes], axis=1) + return boxes + + +def generate_pyramid_anchors(scales, ratios, feature_shapes, feature_strides, + anchor_stride): + """Generate anchors at different levels of a feature pyramid. Each scale + is associated with a level of the pyramid, but each ratio is used in + all levels of the pyramid. + + Returns: + anchors: [N, (y1, x1, y2, x2)]. All generated anchors in one array. Sorted + with the same order of the given scales. So, anchors of scale[0] come + first, then anchors of scale[1], and so on. + """ + # Anchors + # [anchor_count, (y1, x1, y2, x2)] + anchors = [] + for i in range(len(scales)): + anchors.append(generate_anchors(scales[i], ratios, feature_shapes[i], + feature_strides[i], anchor_stride)) + return np.concatenate(anchors, axis=0) + + +############################################################ +# Miscellaneous +############################################################ + +def trim_zeros(x): + """It's common to have tensors larger than the available data and + pad with zeros. This function removes rows that are all zeros. + + x: [rows, columns]. + """ + assert len(x.shape) == 2 + return x[~np.all(x == 0, axis=1)] + + +def compute_matches(gt_boxes, gt_class_ids, gt_masks, + pred_boxes, pred_class_ids, pred_scores, pred_masks, + iou_threshold=0.5, score_threshold=0.0): + """Finds matches between prediction and ground truth instances. + + Returns: + gt_match: 1-D array. For each GT box it has the index of the matched + predicted box. + pred_match: 1-D array. For each predicted box, it has the index of + the matched ground truth box. + overlaps: [pred_boxes, gt_boxes] IoU overlaps. + """ + # Trim zero padding + # TODO: cleaner to do zero unpadding upstream + gt_boxes = trim_zeros(gt_boxes) + gt_masks = gt_masks[..., :gt_boxes.shape[0]] + pred_boxes = trim_zeros(pred_boxes) + pred_scores = pred_scores[:pred_boxes.shape[0]] + # Sort predictions by score from high to low + indices = np.argsort(pred_scores)[::-1] + pred_boxes = pred_boxes[indices] + pred_class_ids = pred_class_ids[indices] + pred_scores = pred_scores[indices] + pred_masks = pred_masks[..., indices] + + # Compute IoU overlaps [pred_masks, gt_masks] + overlaps = compute_overlaps_masks(pred_masks, gt_masks) + + # Loop through predictions and find matching ground truth boxes + match_count = 0 + pred_match = -1 * np.ones([pred_boxes.shape[0]]) + gt_match = -1 * np.ones([gt_boxes.shape[0]]) + for i in range(len(pred_boxes)): + # Find best matching ground truth box + # 1. Sort matches by score + sorted_ixs = np.argsort(overlaps[i])[::-1] + # 2. Remove low scores + low_score_idx = np.where(overlaps[i, sorted_ixs] < score_threshold)[0] + if low_score_idx.size > 0: + sorted_ixs = sorted_ixs[:low_score_idx[0]] + # 3. Find the match + for j in sorted_ixs: + # If ground truth box is already matched, go to next one + if gt_match[j] > 0: + continue + # If we reach IoU smaller than the threshold, end the loop + iou = overlaps[i, j] + if iou < iou_threshold: + break + # Do we have a match? + if pred_class_ids[i] == gt_class_ids[j]: + match_count += 1 + gt_match[j] = i + pred_match[i] = j + break + + return gt_match, pred_match, overlaps + + +def compute_ap(gt_boxes, gt_class_ids, gt_masks, + pred_boxes, pred_class_ids, pred_scores, pred_masks, + iou_threshold=0.5): + """Compute Average Precision at a set IoU threshold (default 0.5). + + Returns: + mAP: Mean Average Precision + precisions: List of precisions at different class score thresholds. + recalls: List of recall values at different class score thresholds. + overlaps: [pred_boxes, gt_boxes] IoU overlaps. + """ + # Get matches and overlaps + gt_match, pred_match, overlaps = compute_matches( + gt_boxes, gt_class_ids, gt_masks, + pred_boxes, pred_class_ids, pred_scores, pred_masks, + iou_threshold) + + # Compute precision and recall at each prediction box step + precisions = np.cumsum(pred_match > -1) / (np.arange(len(pred_match)) + 1) + recalls = np.cumsum(pred_match > -1).astype(np.float32) / len(gt_match) + + # Pad with start and end values to simplify the math + precisions = np.concatenate([[0], precisions, [0]]) + recalls = np.concatenate([[0], recalls, [1]]) + + # Ensure precision values decrease but don't increase. This way, the + # precision value at each recall threshold is the maximum it can be + # for all following recall thresholds, as specified by the VOC paper. + for i in range(len(precisions) - 2, -1, -1): + precisions[i] = np.maximum(precisions[i], precisions[i + 1]) + + # Compute mean AP over recall range + indices = np.where(recalls[:-1] != recalls[1:])[0] + 1 + mAP = np.sum((recalls[indices] - recalls[indices - 1]) * + precisions[indices]) + + return mAP, precisions, recalls, overlaps + + +def compute_ap_range(gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + iou_thresholds=None, verbose=1): + """Compute AP over a range or IoU thresholds. Default range is 0.5-0.95.""" + # Default is 0.5 to 0.95 with increments of 0.05 + iou_thresholds = iou_thresholds or np.arange(0.5, 1.0, 0.05) + + # Compute AP over range of IoU thresholds + AP = [] + for iou_threshold in iou_thresholds: + ap, precisions, recalls, overlaps = \ + compute_ap(gt_box, gt_class_id, gt_mask, + pred_box, pred_class_id, pred_score, pred_mask, + iou_threshold=iou_threshold) + if verbose: + print("AP @{:.2f}:\t {:.3f}".format(iou_threshold, ap)) + AP.append(ap) + AP = np.array(AP).mean() + if verbose: + print("AP @{:.2f}-{:.2f}:\t {:.3f}".format( + iou_thresholds[0], iou_thresholds[-1], AP)) + return AP + + +def compute_recall(pred_boxes, gt_boxes, iou): + """Compute the recall at the given IoU threshold. It's an indication + of how many GT boxes were found by the given prediction boxes. + + pred_boxes: [N, (y1, x1, y2, x2)] in image coordinates + gt_boxes: [N, (y1, x1, y2, x2)] in image coordinates + """ + # Measure overlaps + overlaps = compute_overlaps(pred_boxes, gt_boxes) + iou_max = np.max(overlaps, axis=1) + iou_argmax = np.argmax(overlaps, axis=1) + positive_ids = np.where(iou_max >= iou)[0] + matched_gt_boxes = iou_argmax[positive_ids] + + recall = len(set(matched_gt_boxes)) / gt_boxes.shape[0] + return recall, positive_ids + + +# ## Batch Slicing +# Some custom layers support a batch size of 1 only, and require a lot of work +# to support batches greater than 1. This function slices an input tensor +# across the batch dimension and feeds batches of size 1. Effectively, +# an easy way to support batches > 1 quickly with little code modification. +# In the long run, it's more efficient to modify the code to support large +# batches and getting rid of this function. Consider this a temporary solution +def batch_slice(inputs, graph_fn, batch_size, names=None): + """Splits inputs into slices and feeds each slice to a copy of the given + computation graph and then combines the results. It allows you to run a + graph on a batch of inputs even if the graph is written to support one + instance only. + + inputs: list of tensors. All must have the same first dimension length + graph_fn: A function that returns a TF tensor that's part of a graph. + batch_size: number of slices to divide the data into. + names: If provided, assigns names to the resulting tensors. + """ + if not isinstance(inputs, list): + inputs = [inputs] + + outputs = [] + for i in range(batch_size): + inputs_slice = [x[i] for x in inputs] + output_slice = graph_fn(*inputs_slice) + if not isinstance(output_slice, (tuple, list)): + output_slice = [output_slice] + outputs.append(output_slice) + # Change outputs from a list of slices where each is + # a list of outputs to a list of outputs and each has + # a list of slices + outputs = list(zip(*outputs)) + + if names is None: + names = [None] * len(outputs) + + result = [tf.stack(o, axis=0, name=n) + for o, n in zip(outputs, names)] + if len(result) == 1: + result = result[0] + + return result + + +def download_trained_weights(coco_model_path, verbose=1): + """Download COCO trained weights from Releases. + + coco_model_path: local path of COCO trained weights + """ + if verbose > 0: + print("Downloading pretrained model to " + coco_model_path + " ...") + with urllib.request.urlopen(COCO_MODEL_URL) as resp, open(coco_model_path, 'wb') as out: + shutil.copyfileobj(resp, out) + if verbose > 0: + print("... done downloading pretrained model!") + + +def norm_boxes(boxes, shape): + """Converts boxes from pixel coordinates to normalized coordinates. + boxes: [N, (y1, x1, y2, x2)] in pixel coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [N, (y1, x1, y2, x2)] in normalized coordinates + """ + h, w = shape + scale = np.array([h - 1, w - 1, h - 1, w - 1]) + shift = np.array([0, 0, 1, 1]) + return np.divide((boxes - shift), scale).astype(np.float32) + + +def denorm_boxes(boxes, shape): + """Converts boxes from normalized coordinates to pixel coordinates. + boxes: [N, (y1, x1, y2, x2)] in normalized coordinates + shape: [..., (height, width)] in pixels + + Note: In pixel coordinates (y2, x2) is outside the box. But in normalized + coordinates it's inside the box. + + Returns: + [N, (y1, x1, y2, x2)] in pixel coordinates + """ + h, w = shape + scale = np.array([h - 1, w - 1, h - 1, w - 1]) + shift = np.array([0, 0, 1, 1]) + return np.around(np.multiply(boxes, scale) + shift).astype(np.int32) diff --git a/mask_rcnn/trainer.py b/mask_rcnn/trainer.py new file mode 100644 index 00000000..ea63806a --- /dev/null +++ b/mask_rcnn/trainer.py @@ -0,0 +1,239 @@ +""" +Mask R-CNN +Train on the toy Balloon dataset and implement color splash effect. + +Copyright (c) 2018 Matterport, Inc. +Licensed under the MIT License (see LICENSE for details) +Written by Waleed Abdulla + +------------------------------------------------------------ + +Usage: import the module (see Jupyter notebooks for examples), or run from + the command line as such: + + # Train a new model starting from pre-trained COCO weights + python3 balloon.py train --dataset=/path/to/balloon/dataset --weights=coco + + # Resume training a model that you had trained earlier + python3 balloon.py train --dataset=/path/to/balloon/dataset --weights=last + + # Train a new model starting from ImageNet weights + python3 balloon.py train --dataset=/path/to/balloon/dataset --weights=imagenet + + # Apply color splash to an image + python3 balloon.py splash --weights=/path/to/weights/file.h5 --image= + + # Apply color splash to video using the last weights you trained + python3 balloon.py splash --weights=last --video= +""" + +from mrcnn import model as modellib, utils +from mrcnn.config import Config +import os +import sys +import json +import datetime +import numpy as np +import skimage.draw + +# Root directory of the project +ROOT_DIR = os.path.abspath("../../") + +# Import Mask RCNN +sys.path.append(ROOT_DIR) # To find local version of the library + +# Path to trained weights file +COCO_WEIGHTS_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5") + +# Directory to save logs and model checkpoints, if not provided +# through the command line argument --logs +DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs") + +############################################################ +# Configurations +############################################################ + + +class PPConfig(Config): + """Configuration for training on the toy dataset. + Derives from the base Config class and overrides some values. + """ + # Give the configuration a recognizable name + NAME = "pointless_package" + + # We use a GPU with 12GB memory, which can fit two images. + # Adjust down if you use a smaller GPU. + IMAGES_PER_GPU = 1 + + # Number of classes (including background) + NUM_CLASSES = 1 + 6 # Background + outerbox + innerbox + item_rect + item_rect_slim + item_sq + + # Number of training steps per epoch + STEPS_PER_EPOCH = 100 + + # Skip detections with < 90% confidence + DETECTION_MIN_CONFIDENCE = 0.9 + + +MY_ABS_PATH = "./" +config = PPConfig() +config.display() + +print("Loading Mask R-CNN model...") +my_model_dir = MY_ABS_PATH + 'models/' +model = modellib.MaskRCNN( + mode="training", config=config, model_dir=my_model_dir) + +#n load the weights for COCO +# model.load_weights('./models/mask_rcnn_coco.h5', +# by_name=True, +# exclude=["mrcnn_class_logits", "mrcnn_bbox_fc", "mrcnn_bbox", "mrcnn_mask"]) + +# model.keras_model.summary() + +############################################################ +# Dataset +############################################################ + +class PPDataset(utils.Dataset): + + def load_dataset(self, dataset_dir, subset): + """Load a subset of the Balloon dataset. + dataset_dir: Root directory of the dataset. + subset: Subset to load: train or val + """ + # Add classes. We have only one class to add. + self.add_class("pointless_package", 1, "outerbox") + self.add_class("pointless_package", 2, "innerbox") + self.add_class("pointless_package", 3, "item_sq") + self.add_class("pointless_package", 4, "item_rect") + self.add_class("pointless_package", 5, "item_rect_slim") + self.add_class("pointless_package", 6, "item_circ") + + # Train or validation dataset? + assert subset in ["train", "val"] + dataset_dir = os.path.join(dataset_dir, subset) + + # Load annotations + # VGG Image Annotator (up to version 1.6) saves each image in the form: + # { 'filename': '28503151_5b5b7ec140_b.jpg', + # 'regions': { + # '0': { + # 'region_attributes': {}, + # 'shape_attributes': { + # 'all_points_x': [...], + # 'all_points_y': [...], + # 'name': 'polygon'}}, + # ... more regions ... + # }, + # 'size': 100202 + # } + # We mostly care about the x and y coordinates of each region + # Note: In VIA 2.0, regions was changed from a dict to a list. + annotations = json.load( + open(os.path.join(dataset_dir, "via_region_data.json"))) + annotations = list(annotations.values()) # don't need the dict keys + + # The VIA tool saves images in the JSON even if they don't have any + # annotations. Skip unannotated images. + annotations = [a for a in annotations if a['regions']] + + # Add images + for a in annotations: + # Get the x, y coordinaets of points of the polygons that make up + # the outline of each object instance. These are stores in the + # shape_attributes (see json format above) + # The if condition is needed to support VIA versions 1.x and 2.x. + if type(a['regions']) is dict: + polygons = [r['shape_attributes'] + for r in a['regions'].values()] + else: + polygons = [r['shape_attributes'] for r in a['regions']] + + # load_mask() needs the image size to convert polygons to masks. + # Unfortunately, VIA doesn't include it in JSON, so we must read + # the image. This is only managable since the dataset is tiny. + image_path = os.path.join(dataset_dir, a['filename']) + image = skimage.io.imread(image_path) + height, width = image.shape[:2] + + class_list = [r['region_attributes'] for r in a['regions']] + + self.add_image( + "pointless_package", + image_id=a['filename'], # use file name as a unique image id + path=image_path, + width=width, height=height, + class_list=class_list, + polygons=polygons) + + def load_mask(self, image_id): + """Generate instance masks for an image. + Returns: + masks: A bool array of shape [height, width, instance count] with + one mask per instance. + class_ids: a 1D array of class IDs of the instance masks. + """ + class_ids = list() + # If not a pointless_package dataset image, delegate to parent class. + image_info = self.image_info[image_id] + # if image_info["source"] != "pointless_package": + # return super(self.__class__, self).load_mask(image_id) + + # Convert polygons to a bitmap mask of shape + # [height, width, instance_count] + info = self.image_info[image_id] + # print("\n\n\nIMAGE INFO:", info, "\n\n\n\n") + + for box_type in info['class_list']: + # print(box_type['name']) + class_ids.append(self.class_names.index(str(box_type['name']))) + # print(class_ids) + # print(self.class_names) + + mask = np.zeros([info["height"], info["width"], len(info["polygons"])], + dtype=np.uint8) + for i, p in enumerate(info["polygons"]): + # Get indexes of pixels inside the polygon and set them to 1 + rr, cc = skimage.draw.polygon(p['all_points_y'], p['all_points_x']) + mask[rr, cc, i] = 1 + # Return mask, and array of class IDs of each instance. Since we have + # one class ID only, we return an array of 1s + return mask.astype(np.bool), np.asarray(class_ids, dtype=np.int32) + + def image_reference(self, image_id): + """Return the path of the image.""" + info = self.image_info[image_id] + if info["source"] == "pointless_package": + return info["path"] + else: + super(self.__class__, self).image_reference(image_id) + + +my_dataset_dir = MY_ABS_PATH + 'dataset/' + +train_set = PPDataset() +train_set.load_dataset( + my_dataset_dir, "train") +train_set.prepare() +print('Train: %d' % len(train_set.image_ids)) +# prepare test/val set +test_set = PPDataset() +test_set.load_dataset( + my_dataset_dir, "val") +test_set.prepare() +print('Test: %d' % len(test_set.image_ids)) + + +# train weights (output layers or 'heads') +# train heads with higher lr to speedup the learning +model.train(train_set, test_set, learning_rate=config.LEARNING_RATE, + epochs=5, layers='all') + +history = model.keras_model.history.history + +model.get_trainable_layers() + + +model_path = MY_ABS_PATH + 'models/mask_rcnn_final.h5' +model.keras_model.save_weights(model_path) diff --git a/mask_rcnn/trainer_voc.py b/mask_rcnn/trainer_voc.py new file mode 100644 index 00000000..9da9428b --- /dev/null +++ b/mask_rcnn/trainer_voc.py @@ -0,0 +1,198 @@ +from xml.etree import ElementTree +from os import listdir +from mrcnn.config import Config +from mrcnn import model as modellib +from mrcnn import visualize +import mrcnn +from mrcnn.utils import Dataset +from mrcnn.model import MaskRCNN + +import numpy as np +from numpy import zeros +from numpy import asarray +import colorsys +import argparse +import imutils +import random +import cv2 +import os +import time + +from matplotlib import pyplot +from matplotlib.patches import Rectangle +from keras.models import load_model + +#inherting from Config class + +MY_ABS_PATH = "./" +PP_LABELS=[] + +class PPConfig(Config): + """Configuration for training on the toy dataset. + Derives from the base Config class and overrides some values. + """ + # Give the configuration a recognizable name + NAME = "pointless_package" + + # We use a GPU with 12GB memory, which can fit two images. + # Adjust down if you use a smaller GPU. + IMAGES_PER_GPU = 1 + + # Number of classes (including background) + NUM_CLASSES = 1 + 3 # Background + outerbox + innerbox + product + + # Number of training steps per epoch + STEPS_PER_EPOCH = 100 + + # Skip detections with < 90% confidence + DETECTION_MIN_CONFIDENCE = 0.9 + +config = PPConfig() +config.display() + +print("Loading Mask R-CNN model...") +my_model_dir = MY_ABS_PATH + 'models/' +model = modellib.MaskRCNN( + mode="training", config=config, model_dir=my_model_dir) + +#n load the weights for COCO +model.load_weights('./models/mask_rcnn_coco.h5', + by_name=True, + exclude=["mrcnn_class_logits", "mrcnn_bbox_fc", "mrcnn_bbox", "mrcnn_mask"]) + +model.keras_model.summary() + + +class PPDataset(Dataset): + # load the dataset definitions + def load_dataset(self, dataset_dir, is_train=True): + + # Add classes. We have only one class to add. + self.add_class("dataset", 1, "outerbox") + self.add_class("dataset", 2, "innerbox") + self.add_class("dataset", 3, "product") + + # define data locations for images and annotations + images_dir = dataset_dir + 'images/' + annotations_dir = dataset_dir + 'annotations/' + + # Iterate through all files in the folder to + #add class, images and annotaions + for filename in listdir(images_dir): + + # extract image id + image_id = filename[:-4] + + # skip bad images + if image_id in ['00090']: + continue + # skip all images after 150 if we are building the train set + if is_train and int(image_id[4:]) >= 150: + continue + # skip all images before 150 if we are building the test/val set + if not is_train and int(image_id[4:]) < 150: + continue + + # setting image file + img_path = images_dir + filename + + # setting annotations file + ann_path = annotations_dir + image_id + '.xml' + + # adding images and annotations to dataset + self.add_image('dataset', image_id=image_id, + path=img_path, annotation=ann_path) + + # extract bounding boxes from an annotation file + def extract_boxes(self, filename): + + # load and parse the file + tree = ElementTree.parse(filename) + # get the root of the document + root = tree.getroot() + # extract each bounding box + boxes = list() + for box in root.findall('.//object'): + name = box.find('name').text + xmin = int(box.find('.//bndbox').find('xmin').text) + ymin = int(box.find('.//bndbox').find('ymin').text) + xmax = int(box.find('.//bndbox').find('xmax').text) + ymax = int(box.find('.//bndbox').find('ymax').text) + coors = [xmin, ymin, xmax, ymax, name] + boxes.append(coors) + + # for i, product in enumerate(root.findall('.//object')): + # name = product.find('name').text + # print(name) + + # extract image dimensions + width = int(root.find('.//size/width').text) + height = int(root.find('.//size/height').text) + return boxes, width, height + + # load the masks for an image + """Generate instance masks for an image. + Returns: + masks: A bool array of shape [height, width, instance count] with + one mask per instance. + class_ids: a 1D array of class IDs of the instance masks. + """ + + def load_mask(self, image_id): + # get details of image + info = self.image_info[image_id] + + # define anntation file location + path = info['annotation'] + + # load XML + boxes, w, h = self.extract_boxes(path) + + # create one array for all masks, each on a different channel + masks = zeros([h, w, len(boxes)], dtype=np.uint8) + + # create masks + class_ids = list() + for i in range(len(boxes)): + box = boxes[i] + row_s, row_e = box[1], box[3] + col_s, col_e = box[0], box[2] + masks[row_s:row_e, col_s:col_e, i] = 1 + class_ids.append(self.class_names.index(box[4])) + return masks, asarray(class_ids, dtype=np.int32) + + # load an image reference + #Return the path of the image.""" + def image_reference(self, image_id): + info = self.image_info[image_id] + print(info) + return info['path'] + + +my_dataset_dir = MY_ABS_PATH + 'dataset/' + +train_set = PPDataset() +train_set.load_dataset( + my_dataset_dir, is_train=True) +train_set.prepare() +print('Train: %d' % len(train_set.image_ids)) +# prepare test/val set +test_set = PPDataset() +test_set.load_dataset( + my_dataset_dir, is_train=False) +test_set.prepare() +print('Test: %d' % len(test_set.image_ids)) + + +# train weights (output layers or 'heads') +## train heads with higher lr to speedup the learning +model.train(train_set, test_set, learning_rate=config.LEARNING_RATE, + epochs=5, layers='heads') + +history = model.keras_model.history.history + +model.get_trainable_layers() + + +model_path = MY_ABS_PATH + 'models/mask_rcnn_' + str(time.time()) + '.h5' +model.keras_model.save_weights(model_path) diff --git a/opencv_attempts/BoxDetect.py b/opencv_attempts/BoxDetect.py new file mode 100644 index 00000000..9997f0a2 --- /dev/null +++ b/opencv_attempts/BoxDetect.py @@ -0,0 +1,54 @@ +import numpy as np +import cv2 +import matplotlib.pyplot as plt +import sys + +path = 'images/' +inf_addr = 'box_1' +print(sys.argv) +addr = (path+inf_addr+'.jpg') if len(sys.argv) < 2 else (path + + sys.argv[1]+'.jpg') + +img = cv2.imread(addr,0) + +BINS = 20 +np_hist, _ = np.histogram(img, bins=BINS) +print(np_hist) + +dmin, dmax, _, _ = cv2.minMaxLoc(img) +if np.issubdtype(img.dtype, 'float'): + dmax += np.finfo(img.dtype).eps +else: + dmax += 1 + +cv_hist = cv2.calcHist([img], [0], None, [BINS], [dmin, dmax]).flatten() + +# data = np.reshape(img, (-1, 3)) +# print(data.shape) +# data = np.float32(data) + +# criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0) +# flags = cv2.KMEANS_RANDOM_CENTERS +# compactness, labels, centers = cv2.kmeans(data, 1, None, criteria, 10, flags) + +# centers[0] = centers[0].astype(int) + + +# img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) +# canvas = np.zeros(img.shape) + +# img_blur = cv2.GaussianBlur(img, (7,7), 0) + +# min_color = np.array(centers[0]-20) +# max_color = np.array(centers[0]+20) + +# mask1 = cv2.inRange(img_blur, min_color, max_color) + +plt.plot(np_hist, '-', label='numpy') +plt.plot(cv_hist, '-', label='opencv') + +plt.legend() +plt.show() +# cv2.imshow('',mask1) +cv2.waitKey(0) +cv2.destroyAllWindows() diff --git a/opencv_attempts/countours.py b/opencv_attempts/countours.py new file mode 100644 index 00000000..a3ab2ee0 --- /dev/null +++ b/opencv_attempts/countours.py @@ -0,0 +1,44 @@ +import numpy as np +import cv2 +import matplotlib.pyplot as plt +import sys +import random + +path = 'images/' +inf_addr = 'box_1' +print(sys.argv) +addr = (path+inf_addr+'.jpg') if len(sys.argv) < 2 else (path + sys.argv[1]+'.jpg') + +img = cv2.imread(addr) +img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) +img_gray = cv2.GaussianBlur(img_gray, (7, 7), 3) +canvas = np.zeros(img.shape) + +canny_edges = cv2.Canny(img_gray, 1, 100) +# canny_edges = cv2.dilate(canny_edges, None, iterations=2) +# canny_edges = cv2.erode(canny_edges, None, iterations=2) + +canny_contours, hierarchy = cv2.findContours(canny_edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) + +# cv2.drawContours(canvas, canny_contours, -1, (255, 255, 255), 2) + + +cntsSorted = sorted( + canny_contours, key=lambda x: cv2.contourArea(x), reverse=True) +# cntsSorted = sorted(cnts, key=lambda x: cv2.arcLength(x, True), reverse=True) +cnt_max = cv2.contourArea(cntsSorted[0]) +""" DRAW CONTOUR IMAGE """ +for i, c in enumerate(cntsSorted): + if 0 < cv2.contourArea(c) <= cnt_max: + color = random.randint(1, 1000) % 255 + cv2.drawContours(canvas, cntsSorted[i], -1, (color, color, color), 1) + print("Color:", color, "Area:", cv2.contourArea(c)) + cv2.waitKey(200) + cv2.imshow('CannyCanvas', canvas) + +# cv2.drawContours(img, contours, 5, (0,255,0), 3) + +cv2.imshow('',img) +# cv2.imshow('CannyEdges', canny_edges) +cv2.waitKey(0) +cv2.destroyAllWindows() diff --git a/opencv_attempts/detect.py b/opencv_attempts/detect.py new file mode 100644 index 00000000..404cfec6 --- /dev/null +++ b/opencv_attempts/detect.py @@ -0,0 +1,83 @@ +import numpy as np +import cv2 +import matplotlib.pyplot as plt + +import sys + +path = 'images/' +inf_addr = 'box_1' +print(sys.argv) +addr = (path+inf_addr+'.jpg') if len(sys.argv) < 2 else (path + sys.argv[1]+'.jpg') + + +img = cv2.imread(addr) + +Z = img.reshape((-1, 3)) + +# convert to np.float32 +Z = np.float32(Z) + +# define criteria, number of clusters(K) and apply kmeans() +criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0) +K = 3 +ret, label, center = cv2.kmeans( + Z, K, None, criteria, 10, cv2.KMEANS_RANDOM_CENTERS) + +# Now convert back into uint8, and make original image +center = np.uint8(center) +res = center[label.flatten()] +res2 = res.reshape((img.shape)) + + +img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) +# img_gray = res2 + +# kernel = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]]) +# img_gray = cv2.filter2D(img_gray, -1, kernel) + +img_gray = cv2.GaussianBlur(img_gray, (7, 7), 3) +img_gray2 = img_gray.copy() + +canvas = np.zeros(shape=img.shape) + +edged = cv2.Canny(img_gray, 0, 65) +edged = cv2.dilate(edged, None, iterations=1) +edged = cv2.erode(edged, None, iterations=1) + +ret, thresh = cv2.threshold(img_gray2, 127, 255, 0) + +cnts, hierarchy = cv2.findContours( + edged.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + +# contours, _ = cv2.findContours( +# thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + +cv2.drawContours(img, cnts, -1, (0, 255, 0), 3) +ddepth = -1 +ind = 2 +kernel_size = 3 + 2 * (ind % 5) +kernel = np.ones((kernel_size, kernel_size), dtype=np.float32) +kernel /= (kernel_size * kernel_size) + +dst = cv2.filter2D(img, ddepth, kernel) + +# cnts = cnts[0] if imutils.is_cv2() else cnts[1] +cntsSorted = sorted(cnts, key=lambda x: cv2.contourArea(x), reverse=True) +# cntsSorted = sorted(cnts, key=lambda x: cv2.arcLength(x, True), reverse=True) + +""" DRAW CONTOUR IMAGE """ +for i, c in enumerate(cntsSorted): + cv2.drawContours(canvas, c, -1, (255, 255, 255), 2) + print(cv2.contourArea(c)) + cv2.waitKey(100) + cv2.imshow('Canny',canvas) + +# cv2.imshow("Custom Filter", dst) + +# cv2.imshow('', img) +# cv2.imshow('Threshold Contours', thresh) +cv2.imshow('Color Quantization', res2) +cv2.waitKey(0) +cv2.destroyAllWindows() + diff --git a/opencv_attempts/images/1.jpg b/opencv_attempts/images/1.jpg new file mode 100644 index 00000000..1c2ccd99 Binary files /dev/null and b/opencv_attempts/images/1.jpg differ diff --git a/opencv_attempts/images/10.jpg b/opencv_attempts/images/10.jpg new file mode 100644 index 00000000..928f06cb Binary files /dev/null and b/opencv_attempts/images/10.jpg differ diff --git a/opencv_attempts/images/11.jpg b/opencv_attempts/images/11.jpg new file mode 100644 index 00000000..a0ae6cf3 Binary files /dev/null and b/opencv_attempts/images/11.jpg differ diff --git a/opencv_attempts/images/12.jpg b/opencv_attempts/images/12.jpg new file mode 100644 index 00000000..1640f87e Binary files /dev/null and b/opencv_attempts/images/12.jpg differ diff --git a/opencv_attempts/images/13.jpg b/opencv_attempts/images/13.jpg new file mode 100644 index 00000000..a3dca8f6 Binary files /dev/null and b/opencv_attempts/images/13.jpg differ diff --git a/opencv_attempts/images/14.jpg b/opencv_attempts/images/14.jpg new file mode 100644 index 00000000..d7c8faff Binary files /dev/null and b/opencv_attempts/images/14.jpg differ diff --git a/opencv_attempts/images/15.jpg b/opencv_attempts/images/15.jpg new file mode 100644 index 00000000..fd667ea6 Binary files /dev/null and b/opencv_attempts/images/15.jpg differ diff --git a/opencv_attempts/images/16.jpg b/opencv_attempts/images/16.jpg new file mode 100644 index 00000000..782024eb Binary files /dev/null and b/opencv_attempts/images/16.jpg differ diff --git a/opencv_attempts/images/17.jpg b/opencv_attempts/images/17.jpg new file mode 100644 index 00000000..2c820657 Binary files /dev/null and b/opencv_attempts/images/17.jpg differ diff --git a/opencv_attempts/images/18.jpg b/opencv_attempts/images/18.jpg new file mode 100644 index 00000000..33f123af Binary files /dev/null and b/opencv_attempts/images/18.jpg differ diff --git a/opencv_attempts/images/19.jpg b/opencv_attempts/images/19.jpg new file mode 100644 index 00000000..98fa6ff2 Binary files /dev/null and b/opencv_attempts/images/19.jpg differ diff --git a/opencv_attempts/images/2.jpg b/opencv_attempts/images/2.jpg new file mode 100644 index 00000000..c7e8400c Binary files /dev/null and b/opencv_attempts/images/2.jpg differ diff --git a/opencv_attempts/images/20.jpg b/opencv_attempts/images/20.jpg new file mode 100644 index 00000000..288a6a96 Binary files /dev/null and b/opencv_attempts/images/20.jpg differ diff --git a/opencv_attempts/images/21.jpg b/opencv_attempts/images/21.jpg new file mode 100644 index 00000000..47eb3ad8 Binary files /dev/null and b/opencv_attempts/images/21.jpg differ diff --git a/opencv_attempts/images/3.jpg b/opencv_attempts/images/3.jpg new file mode 100644 index 00000000..d11c31f8 Binary files /dev/null and b/opencv_attempts/images/3.jpg differ diff --git a/opencv_attempts/images/4.jpg b/opencv_attempts/images/4.jpg new file mode 100644 index 00000000..0c651d1e Binary files /dev/null and b/opencv_attempts/images/4.jpg differ diff --git a/opencv_attempts/images/5.jpg b/opencv_attempts/images/5.jpg new file mode 100644 index 00000000..290b658e Binary files /dev/null and b/opencv_attempts/images/5.jpg differ diff --git a/opencv_attempts/images/6.jpg b/opencv_attempts/images/6.jpg new file mode 100644 index 00000000..c2462720 Binary files /dev/null and b/opencv_attempts/images/6.jpg differ diff --git a/opencv_attempts/images/7.jpg b/opencv_attempts/images/7.jpg new file mode 100644 index 00000000..82052589 Binary files /dev/null and b/opencv_attempts/images/7.jpg differ diff --git a/opencv_attempts/images/8.jpg b/opencv_attempts/images/8.jpg new file mode 100644 index 00000000..b5133d9c Binary files /dev/null and b/opencv_attempts/images/8.jpg differ diff --git a/opencv_attempts/images/9.jpg b/opencv_attempts/images/9.jpg new file mode 100644 index 00000000..66e89582 Binary files /dev/null and b/opencv_attempts/images/9.jpg differ diff --git a/opencv_attempts/watershed.py b/opencv_attempts/watershed.py new file mode 100644 index 00000000..0c84051b --- /dev/null +++ b/opencv_attempts/watershed.py @@ -0,0 +1,39 @@ +import numpy as np +import cv2 as cv +from matplotlib import pyplot as plt +import sys + +path = 'images/' +inf_addr = 'box_1' +print(sys.argv) +addr = (path+inf_addr+'.jpg') if len(sys.argv) < 2 else (path + sys.argv[1]) + +img = cv.imread(addr) +gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY) +ret, thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV+cv.THRESH_OTSU) + +# noise removal +kernel = np.ones((3, 3), np.uint8) +opening = cv.morphologyEx(thresh, cv.MORPH_OPEN, kernel, iterations=2) +# sure background area +sure_bg = cv.dilate(opening, kernel, iterations=3) +# Finding sure foreground area +dist_transform = cv.distanceTransform(opening, cv.DIST_L2, 5) +ret, sure_fg = cv.threshold(dist_transform, 0.7*dist_transform.max(), 255, 0) +# Finding unknown region +sure_fg = np.uint8(sure_fg) +unknown = cv.subtract(sure_bg, sure_fg) + +# Marker labelling +ret, markers = cv.connectedComponents(sure_fg) +# Add one to all labels so that sure background is not 0, but 1 +markers = markers+1 +# Now, mark the region of unknown with zero +markers[unknown == 255] = 0 + +markers = cv.watershed(img, markers) +img[markers == -1] = [255, 0, 0] + +cv.imshow('', img) +cv.waitKey(0) +cv.destroyAllWindows() diff --git a/yoosuf_scripts/rename.py b/yoosuf_scripts/rename.py new file mode 100644 index 00000000..a2d3f0f0 --- /dev/null +++ b/yoosuf_scripts/rename.py @@ -0,0 +1,70 @@ + +''' +Simple python script to rename all files to rename any files to +ECS 193A naming convention. +''' + + +import argparse + +import io +import os +import shutil +import random + +def parse_args(): + ''' + Get command line args. + ''' + + parser = argparse.ArgumentParser() + parser.add_argument('--dir', required=True, help='path to \ + directory with images', dest='dir') + + parser.add_argument('--newdir', required=True, help='path to \ + new directory with renamed images', dest='new_dir') + + parser.add_argument('--start', required=True, help='naming \ + starting number', dest='num_start', type=int) + + args = parser.parse_args() + + return args + +def rename(files, nums_start): + ''' + Renames the files in a directory. + ''' + +def main(): + + args = parse_args() + + try: + files = os.listdir(args.dir) + except NotADirectoryError: + print("No such directory ", args.dir) + raise + + new_directory = args.new_dir + + # randomize the files + random.shuffle(files) + + #os.mkdir(new_directory) + shutil.copytree(args.dir, new_directory) + + current_index = args.num_start + + for file in files: + full_name = os.path.join(new_directory, file) + new_name = os.path.join(new_directory, 'IMG_' + \ + str(current_index) + '.jpg') + + current_index += 1 + os.rename(full_name, new_name) + + print("Rename successful!") + +if __name__== "__main__": + main() diff --git a/yoosuf_scripts/resize.py b/yoosuf_scripts/resize.py new file mode 100644 index 00000000..2d92386b --- /dev/null +++ b/yoosuf_scripts/resize.py @@ -0,0 +1,88 @@ +import argparse +import io +import os +import sys +import shutil +import cv2 + +width = 300 +accpted_extensions = ['.jpg', '.jpeg', '.png', '.gif'] + +description = "Simple script that takes a directory of images and resizes them to width of " + str(width) + "px while still maintaining the aspect ratio." + +def parse_args(): + parser = argparse.ArgumentParser(description=description) + parser.add_argument('--src', required=True, help='Path to the directory with source images', dest='src') + parser.add_argument('--dest', required=False, help='[Optional] New target directory. If supplied, a new directory will be created and all images will be copied and resized in this directory.', dest='dest') + + return parser.parse_args() + +def query_yes_no(): + print("\n* You have NOT specified a destination.\nThe script will rewrite images to the same directory and original images will not be saved.\nAre you sure you want to continue?\n") + + yes = {'yes', 'y', 'ye', ''} + no = {'no', 'n'} + print("Please respond with 'y' or 'n':", end=' ') + while True: + choice = input().lower() + if choice in yes: + return True + elif choice in no: + return False + else: + print("Please respond with 'y' or 'n':", end=' ') + +def main(): + args = parse_args() + source_dir = args.src # source directory name + dest_dir = args.dest # destination directory name + op_dir = source_dir # operation directory + + if os.path.isdir(source_dir) == False: + print("Please enter a valid source directory.") + raise NotADirectoryError + + if dest_dir == None and query_yes_no() == False: + print("Script aborted.") + raise SystemExit + + if dest_dir != None: + """ Copy all files from source to destination directory """ + shutil.copytree(source_dir, dest_dir) + op_dir = dest_dir + + try: + """ Get all files in the copied directory """ + files = os.listdir(op_dir) + except NotADirectoryError: + print("Invalid source directory", op_dir) + raise + + """ Check if absolute path """ + file_path = '' + if os.path.isabs(op_dir) == False: + file_path = './' + op_dir + '/' + + resized = 0 + total_img = 0 + + for file in files: + img_name = file_path+file + if os.path.splitext(img_name)[1] in accpted_extensions: # check if image is jpg, png + img = cv2.imread(img_name) + orig_dim = img.shape + total_img += 1 + if orig_dim[0] < 301: + print("Did not resize", file, "with dimensions", orig_dim[:2]) + continue + new_dim = (int((width/orig_dim[0]) * orig_dim[1]), width) + resized_img = cv2.resize(img, new_dim, interpolation=cv2.INTER_AREA) + cv2.imwrite(img_name, resized_img) + resized += 1 + print("Resized", file, "from", orig_dim[:2], "->", resized_img.shape[:2]) + + print(f"\nImages Resized: {resized} out of {total_img} images.") + print("Done!") + +if __name__ == "__main__": + main()