From 6811c7145c2657d08b304b85f793b69ae01f115b Mon Sep 17 00:00:00 2001 From: mlee3142 Date: Mon, 18 Nov 2019 16:50:44 -0500 Subject: [PATCH 1/4] tensorflow.keras KerasWrapper --- model_tools/activations/keras.py | 29 ++++++++++++++++++--------- model_tools/activations/tensorflow.py | 2 +- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/model_tools/activations/keras.py b/model_tools/activations/keras.py index 34c8774..3c03838 100644 --- a/model_tools/activations/keras.py +++ b/model_tools/activations/keras.py @@ -4,6 +4,7 @@ from model_tools.activations.core import ActivationsExtractorHelper +import tensorflow as tf class KerasWrapper: def __init__(self, model, preprocessing, identifier=None, *args, **kwargs): @@ -18,6 +19,9 @@ def __init__(self, model, preprocessing, identifier=None, *args, **kwargs): *args, **kwargs) self._extractor.insert_attrs(self) + def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention + return self._extractor(*args, **kwargs) + @property def identifier(self): return self._extractor.identifier @@ -26,20 +30,27 @@ def identifier(self): def identifier(self, value): self._extractor.identifier = value - def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention - return self._extractor(*args, **kwargs) - def get_activations(self, images, layer_names): - from keras import backend as K + + """ + param images: a list of image paths + param layer_names: a list of layer names + """ + from tensorflow.keras import backend as K + print(images.shape) + input_tensor = self._model.input layers = [layer for layer in self._model.layers if layer.name in layer_names] layers = sorted(layers, key=lambda layer: layer_names.index(layer.name)) + if 'logits' in layer_names: layers.insert(layer_names.index('logits'), self._model.layers[-1]) + assert len(layers) == len(layer_names) layer_out_tensors = [layer.output for layer in layers] - functor = K.function([input_tensor] + [K.learning_phase()], layer_out_tensors) # evaluate all tensors at once - layer_outputs = functor([images, 0.]) # 0 to signal testing phase + functor = K.function([input_tensor], layer_out_tensors) # evaluate all tensors at once + K.set_learning_phase(0) # 0 to signal testing phase + layer_outputs = functor([images]) return OrderedDict([(layer_name, layer_output) for layer_name, layer_output in zip(layer_names, layer_outputs)]) def __repr__(self): @@ -62,7 +73,7 @@ def load_images(image_filepaths, image_size): def load_image(image_filepath): - from keras.preprocessing import image + from tensorflow.keras.preprocessing import image img = image.load_img(image_filepath) x = image.img_to_array(img) return x @@ -70,7 +81,7 @@ def load_image(image_filepath): def scale_image(img, image_size): from PIL import Image - from keras.preprocessing import image + from tensorflow.keras.preprocessing import image img = Image.fromarray(img.astype(np.uint8)) img = img.resize((image_size, image_size)) img = image.img_to_array(img) @@ -79,6 +90,6 @@ def scale_image(img, image_size): def preprocess(image_filepaths, image_size, *args, **kwargs): # only a wrapper to avoid top-level keras imports - from keras.applications.imagenet_utils import preprocess_input + from tensorflow.keras.applications.imagenet_utils import preprocess_input images = load_images(image_filepaths, image_size=image_size) return preprocess_input(images, *args, **kwargs) diff --git a/model_tools/activations/tensorflow.py b/model_tools/activations/tensorflow.py index b79cf81..ecc3dd6 100644 --- a/model_tools/activations/tensorflow.py +++ b/model_tools/activations/tensorflow.py @@ -54,7 +54,7 @@ def get_activations(self, images, layer_names): def load_image(image_filepath): import tensorflow as tf - image = tf.read_file(image_filepath) + image = tf.io.read_file(image_filepath) image = tf.image.decode_png(image, channels=3) return image From 9d050d719966c40729935db414c6d4eb06114af0 Mon Sep 17 00:00:00 2001 From: mlee3142 Date: Tue, 19 Nov 2019 09:47:51 -0500 Subject: [PATCH 2/4] Revert "tensorflow.keras KerasWrapper" This reverts commit 6811c714 --- model_tools/activations/keras.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/model_tools/activations/keras.py b/model_tools/activations/keras.py index 3c03838..34c8774 100644 --- a/model_tools/activations/keras.py +++ b/model_tools/activations/keras.py @@ -4,7 +4,6 @@ from model_tools.activations.core import ActivationsExtractorHelper -import tensorflow as tf class KerasWrapper: def __init__(self, model, preprocessing, identifier=None, *args, **kwargs): @@ -19,9 +18,6 @@ def __init__(self, model, preprocessing, identifier=None, *args, **kwargs): *args, **kwargs) self._extractor.insert_attrs(self) - def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention - return self._extractor(*args, **kwargs) - @property def identifier(self): return self._extractor.identifier @@ -30,27 +26,20 @@ def identifier(self): def identifier(self, value): self._extractor.identifier = value - def get_activations(self, images, layer_names): - - """ - param images: a list of image paths - param layer_names: a list of layer names - """ - from tensorflow.keras import backend as K - print(images.shape) + def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention + return self._extractor(*args, **kwargs) + def get_activations(self, images, layer_names): + from keras import backend as K input_tensor = self._model.input layers = [layer for layer in self._model.layers if layer.name in layer_names] layers = sorted(layers, key=lambda layer: layer_names.index(layer.name)) - if 'logits' in layer_names: layers.insert(layer_names.index('logits'), self._model.layers[-1]) - assert len(layers) == len(layer_names) layer_out_tensors = [layer.output for layer in layers] - functor = K.function([input_tensor], layer_out_tensors) # evaluate all tensors at once - K.set_learning_phase(0) # 0 to signal testing phase - layer_outputs = functor([images]) + functor = K.function([input_tensor] + [K.learning_phase()], layer_out_tensors) # evaluate all tensors at once + layer_outputs = functor([images, 0.]) # 0 to signal testing phase return OrderedDict([(layer_name, layer_output) for layer_name, layer_output in zip(layer_names, layer_outputs)]) def __repr__(self): @@ -73,7 +62,7 @@ def load_images(image_filepaths, image_size): def load_image(image_filepath): - from tensorflow.keras.preprocessing import image + from keras.preprocessing import image img = image.load_img(image_filepath) x = image.img_to_array(img) return x @@ -81,7 +70,7 @@ def load_image(image_filepath): def scale_image(img, image_size): from PIL import Image - from tensorflow.keras.preprocessing import image + from keras.preprocessing import image img = Image.fromarray(img.astype(np.uint8)) img = img.resize((image_size, image_size)) img = image.img_to_array(img) @@ -90,6 +79,6 @@ def scale_image(img, image_size): def preprocess(image_filepaths, image_size, *args, **kwargs): # only a wrapper to avoid top-level keras imports - from tensorflow.keras.applications.imagenet_utils import preprocess_input + from keras.applications.imagenet_utils import preprocess_input images = load_images(image_filepaths, image_size=image_size) return preprocess_input(images, *args, **kwargs) From 228a9d28e26bd60047aa1aefd53606317f9e6d26 Mon Sep 17 00:00:00 2001 From: mlee3142 Date: Tue, 19 Nov 2019 10:00:11 -0500 Subject: [PATCH 3/4] added wrapper for tf(>=2.0.0).keras model --- model_tools/activations/__init__.py | 1 + model_tools/activations/tfkeras.py | 119 ++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 model_tools/activations/tfkeras.py diff --git a/model_tools/activations/__init__.py b/model_tools/activations/__init__.py index e5e8e8c..ba6fac1 100644 --- a/model_tools/activations/__init__.py +++ b/model_tools/activations/__init__.py @@ -1,3 +1,4 @@ from model_tools.activations.keras import KerasWrapper, preprocess as preprocess_keras from model_tools.activations.pytorch import PytorchWrapper, preprocess_images as preprocess_pytorch from model_tools.activations.tensorflow import TensorflowWrapper, TensorflowSlimWrapper +from model_tools.activations.tfkeras import TFKerasWrapper, resnet_preprocessing diff --git a/model_tools/activations/tfkeras.py b/model_tools/activations/tfkeras.py new file mode 100644 index 0000000..c4ecdbb --- /dev/null +++ b/model_tools/activations/tfkeras.py @@ -0,0 +1,119 @@ +from collections import OrderedDict +import numpy as np +from model_tools.activations.core import ActivationsExtractorHelper +import tensorflow as tf + + +class TFKerasWrapper: + """ + A wrapper for the Model class created from tensorflow(>=2.0.0).keras + """ + def __init__(self, + model, + preprocessing, + identifier=None, + *args, **kwargs): + """ + :param model: a tf.keras model with a function `preprocess_input` + that will later be called on the loaded numpy image + """ + self._model = model + identifier = identifier or model.name + self._extractor = ActivationsExtractorHelper( + identifier=identifier, get_activations=self.get_activations, preprocessing=preprocessing, + *args, **kwargs) + self._extractor.insert_attrs(self) + + def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention + return self._extractor(*args, **kwargs) + + @property + def identifier(self): + return self._extractor.identifier + + @identifier.setter + def identifier(self, value): + self._extractor.identifier = value + + def get_activations(self, images, layer_names): + + """ + param images: a list of image paths + param layer_names: a list of layer names + """ + from tensorflow.keras import backend as K + + input_tensor = self._model.input + layers = [layer for layer in self._model.layers if layer.name in layer_names] + layers = sorted(layers, key=lambda layer: layer_names.index(layer.name)) + + if 'logits' in layer_names: + layers.insert(layer_names.index('logits'), self._model.layers[-1]) + + assert len(layers) == len(layer_names) + layer_out_tensors = [layer.output for layer in layers] + functor = K.function([input_tensor], layer_out_tensors) # evaluate all tensors at once + K.set_learning_phase(0) # 0 to signal testing phase + layer_outputs = functor([images]) + return OrderedDict([(layer_name, layer_output) for layer_name, layer_output in zip(layer_names, layer_outputs)]) + + def __repr__(self): + return repr(self._model) + + def graph(self): + import networkx as nx + g = nx.DiGraph() + for layer in self._model.layers: + g.add_node(layer.name, object=layer, type=type(layer)) + for outbound_node in layer._outbound_nodes: + g.add_edge(layer.name, outbound_node.outbound_layer.name) + return g + + + +def tfkeras_load_images(image_paths): + """ + :param image_paths: list of strings of len B + return tf.Tensor of [B, H, W, 3] of dtype tf.uint8 + """ + + def load_image(path): + """ + param path: tf.Tensor (1,) + """ + blob = tf.io.read_file(path[0]) + im = tf.image.decode_png(blob, channels=3) + return im + + def load_images(paths): + images = tf.map_fn(load_image, elems=paths, dtype=tf.uint8) + return images + + images = list(map(lambda s: [s], image_paths)) + images = tf.constant(images) + images = load_images(images) # [b, h, w, 3], dtype tf.uint8 + + return images + + +def resnet_preprocessing(image_paths, + image_size=224): + _R_MEAN = 123.68 + _G_MEAN = 116.78 + _B_MEAN = 103.94 + CHANNEL_MEANS = [_R_MEAN, _G_MEAN, _B_MEAN] + + images = tfkeras_load_images(image_paths) + + # Execute resizing + images = tf.image.resize( + images, + (image_size, image_size), + method=tf.image.ResizeMethod.BILINEAR, + preserve_aspect_ratio=False, + antialias=False, + ) + + images = images - CHANNEL_MEANS + + return images From 4cae34a671f8cc6b06947ec0474b685e1cad1c26 Mon Sep 17 00:00:00 2001 From: mlee3142 Date: Tue, 19 Nov 2019 10:03:19 -0500 Subject: [PATCH 4/4] Revert "tensorflow.keras KerasWrapper" This reverts commit 6811c714 --- model_tools/activations/tensorflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model_tools/activations/tensorflow.py b/model_tools/activations/tensorflow.py index ecc3dd6..b79cf81 100644 --- a/model_tools/activations/tensorflow.py +++ b/model_tools/activations/tensorflow.py @@ -54,7 +54,7 @@ def get_activations(self, images, layer_names): def load_image(image_filepath): import tensorflow as tf - image = tf.io.read_file(image_filepath) + image = tf.read_file(image_filepath) image = tf.image.decode_png(image, channels=3) return image