grad_cam.py

from keras.applications.vgg16 import preprocess_input, decode_predictions
from keras.preprocessing import image
from keras.layers.core import Lambda
from keras.models import Model, load_model
from tensorflow.python.framework import ops
from imutils.paths import list_images
import keras.backend as K
import tensorflow as tf
import numpy as np
import keras
import argparse
import sys
import os
import cv2

def target_category_loss(x, category_index, nb_classes):
    return tf.multiply(x, K.one_hot([category_index], nb_classes))

def target_category_loss_output_shape(input_shape):
    return input_shape

def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)

def load_image(img_path, target_size):
    img = image.load_img(img_path, target_size=target_size)
    x = image.img_to_array(img)
    x = np.expand_dims(x, axis=0)
    x = preprocess_input(x)
    return x

def register_gradient():
    if "GuidedBackProp" not in ops._gradient_registry._registry:
        @ops.RegisterGradient("GuidedBackProp")
        def _GuidedBackProp(op, grad):
            dtype = op.inputs[0].dtype
            return grad * tf.cast(grad > 0., dtype) * \
                tf.cast(op.inputs[0] > 0., dtype)

def compile_saliency_function(model, activation_layer='block5_conv3'):
    input_img = model.input
    layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
    layer_output = layer_dict[activation_layer].output
    max_output = K.max(layer_output, axis=3)
    saliency = K.gradients(K.sum(max_output), input_img)[0]
    return K.function([input_img, K.learning_phase()], [saliency])

def modify_backprop(model, name, model_filename):
    g = tf.get_default_graph()
    with g.gradient_override_map({'Relu': name}):

        # get layers that have an activation
        layer_dict = [layer for layer in model.layers[1:]
                      if hasattr(layer, 'activation')]

        # replace relu activation
        for layer in layer_dict:
            if layer.activation == keras.activations.relu:
                layer.activation = tf.nn.relu

        # re-instanciate a new model
        new_model = load_model(model_filename)
    return new_model

def deprocess_image(x):
    '''
    Same normalization as in:
    https://github.com/fchollet/keras/blob/master/examples/conv_filter_visualization.py
    '''
    if np.ndim(x) > 3:
        x = np.squeeze(x)
    # normalize tensor: center on 0., ensure std is 0.1
    x -= x.mean()
    x /= (x.std() + 1e-5)
    x *= 0.1

    # clip to [0, 1]
    x += 0.5
    x = np.clip(x, 0, 1)

    # convert to RGB array
    x *= 255
    if K.image_dim_ordering() == 'th':
        x = x.transpose((1, 2, 0))
    x = np.clip(x, 0, 255).astype('uint8')
    return x

def grad_cam(input_model, image, category_index, layer_name, target_size, nb_classes=2):
    target_layer = lambda x: target_category_loss(x, category_index, nb_classes)

    last = Lambda(target_layer, output_shape=target_category_loss_output_shape)(input_model.output)
    model = Model(inputs=input_model.input, outputs=last)

    loss = K.sum(model.layers[-1].output)
    conv_output = [l for l in model.layers if l.name == layer_name][0].output
    print("conv_output {}".format(conv_output))
    #conv_output =  input_model.get_layer(name=layer_name).output
    grads = normalize(K.gradients(loss, conv_output)[0])
    gradient_function = K.function([model.inputs[0]], [conv_output, grads])   

    output, grads_val = gradient_function([image])
    output, grads_val = output[0, :], grads_val[0, :, :, :]

    weights = np.mean(grads_val, axis = (0, 1))
    cam = np.ones(output.shape[0 : 2], dtype = np.float32)

    for i, w in enumerate(weights):
        cam += w * output[:, :, i]

    cam = cv2.resize(cam, target_size)
    cam = np.maximum(cam, 0)
    heatmap = cam / np.max(cam)

    #Return to BGR [0..255] from the preprocessed image
    image = image[0, :]
    image -= np.min(image)
    image = np.minimum(image, 255)

    cam = cv2.applyColorMap(np.uint8(255*heatmap), cv2.COLORMAP_JET)
    cam = np.float32(cam) + np.float32(image)
    cam = 255 * cam / np.max(cam)
    return np.uint8(cam), heatmap

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--image", required=True, help="The image for which the gradcam images should be calculated")
    parser.add_argument("-m", "--model", required=True, help="The model to visualize")
    parser.add_argument("-o", "--output-directory", required=True, help="The output directory")
    parser.add_argument("-t", "--target-size", default=(187,187), nargs='+', type=int, help="The target size for the network input")
    args = vars(parser.parse_args())

    image_filename = args["image"]
    model = load_model(args["model"])
    target_size = tuple(args["target_size"])
    print("Loading file: {}".format(image_filename))
    preprocessed_input = load_image(image_filename, target_size=target_size)
    predictions = model.predict(preprocessed_input)
    predicted_class = np.argmax(predictions)
    print("Predicted class: {}".format(predicted_class))

    cam, heatmap = grad_cam(model, preprocessed_input, predicted_class, "block5_conv3", target_size)
    output_filename, _ = os.path.splitext(image_filename)
    output_filename = os.path.join(args["output_directory"], os.path.basename(output_filename))
    cv2.imwrite("{}_gradcam.png".format(output_filename), cam)

    register_gradient()
    guided_model = modify_backprop(model, 'GuidedBackProp', args["model"])
    saliency_fn = compile_saliency_function(guided_model)
    saliency = saliency_fn([preprocessed_input, 0])
    gradcam = saliency[0] * heatmap[..., np.newaxis]
    cv2.imwrite("{}_guided_gradcam.png".format(output_filename), deprocess_image(gradcam))

if __name__ == "__main__":
    main()