From 2c0457aa704b85378d9f86b06ce2acb848a25663 Mon Sep 17 00:00:00 2001 From: maxthibeau Date: Wed, 24 Apr 2019 10:44:31 -0600 Subject: [PATCH 1/3] Create neural_style_transfer.py I <3 git --- neural_style_transfer.py | 191 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 neural_style_transfer.py diff --git a/neural_style_transfer.py b/neural_style_transfer.py new file mode 100644 index 0000000..e54be21 --- /dev/null +++ b/neural_style_transfer.py @@ -0,0 +1,191 @@ +import keras.preprocessing as kp +import matplotlib.pyplot as plt +import numpy as np + +import vgg +import keras.backend as K +import keras.layers as kl +import keras.models as km +import tensorflow as tf + +import scipy.optimize as sio + +content_path = './main_hall.jpg' +style_path = './starry_night.jpg' + +# Load image to get geometry +temp_img = kp.image.load_img(content_path) +width,height = temp_img.size + +# fix the number of rows, while adapting the aspect ratio +img_rows = 400 +img_cols = int(width * img_rows / height) + +# Load content image +content_img = kp.image.load_img(content_path, target_size=(img_rows, img_cols)) +content_img = kp.image.img_to_array(content_img) +# plt.figure() +# plt.imshow(content_img.astype(int)) + +# Load style image +style_img = kp.image.load_img(style_path, target_size=(img_rows, img_cols)) +style_img = kp.image.img_to_array(style_img) +# plt.figure() +# plt.imshow(style_img.astype(int)) + +# plt.show() + +content_img[:, :, 0] -= 103.939 +content_img[:, :, 1] -= 116.779 +content_img[:, :, 2] -= 123.68 +content_img = np.expand_dims(content_img, axis=0) + +style_img[:, :, 0] -= 103.939 +style_img[:, :, 1] -= 116.779 +style_img[:, :, 2] -= 123.68 +style_img = np.expand_dims(style_img, axis=0) + +# Note that we'll be working quite a bit with the TensorFlow objects that underlie Keras +content_model_input = kl.Input(tensor=K.tf.Variable(content_img)) + +content_base_model = vgg.VGG19(input_tensor=content_model_input) +evaluator = K.function([content_base_model.input],[content_base_model.output]) +feature_maps = evaluator([content_img]) +# plt.imshow(feature_maps[0][0,:,:,500]) +# plt.show() + +# Define the layer outputs that we are interested in +content_layers = ['block4_conv2'] + +# Get the tensor outputs of those layers +content_outputs = [content_base_model.get_layer(n).output for n in content_layers] + +# Instantiate a new model with those outputs as outputs +content_model = km.Model(inputs=content_base_model.inputs,outputs=[content_base_model.get_layer(n).output for n in content_layers]) + +# This is not used any further, it's just for visualizing the features +evaluator = K.function([content_model.input],[content_model.output]) +feature_maps = evaluator([content_img]) +#plt.imshow(feature_maps[0][0,:,:,125]) +#plt.show() + +# Please call this second network 'style_model' + +#! Change me +style_layers = ['block1_relu1', 'block2_relu1', 'block3_relu1', 'block4_relu1', 'block5_relu1'] + +style_base_model = vgg.VGG19(input_tensor=kl.Input(tensor=K.tf.Variable(style_img))) + +#! Change me +style_model = km.Model(inputs=style_base_model.inputs,outputs=[style_base_model.get_layer(n).output for n in style_layers]) + +blended_model_input = kl.Input(shape=content_img.shape[1:]) + +# Please call this third network 'blend_model' + +blend_layers = ['block4_conv2', 'block1_relu1', 'block2_relu1', 'block3_relu1', 'block4_relu1', 'block5_relu1'] + +blend_base_model = vgg.VGG19(input_tensor=blended_model_input) + +#! Change me +blend_model = km.Model(inputs=blend_base_model.inputs,outputs=[blend_base_model.get_layer(n).output for n in blend_layers]) + +# Separate the model outputs into those intended for comparison with the content layer and the style layer +blend_content_outputs = [blend_model.outputs[0]] +blend_style_outputs = blend_model.outputs[1:] +def content_layer_loss(Fp, Fx): + #! Change me + _,h,w,d = Fp.get_shape().as_list() + constant = 1/(2*(w*h)**.5*d**.5) + suum = tf.reduce_sum(np.sum((Fx[0,:,:,l] - Fp[0,:,:,l])**2 for l in range(0, d))) + loss = constant*suum + return loss + +content_loss = content_layer_loss(content_model.output,blend_content_outputs[0]) + +# The correct output of this function is 195710720.0 +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +content_loss_evaluator = K.function([blend_model.input],[content_loss]) +# print(content_loss_evaluator([input_img])) + +def gram_matrix(f, M, N): + # Accepts a (height,width,depth)-sized feature map, + # reshapes to (M,N), then computes the inner product + reshaped_f = kl.Reshape((M, N))(f)[0] + reshaped_f_t = K.transpose(reshaped_f) + gram_matrix = K.dot(reshaped_f_t, reshaped_f) + # !Change me + return gram_matrix + +# For a correctly implemented gram_matrix, the following code will produce 113934860.0 +fmap = content_model.output +_,h,w,d = fmap.get_shape() +M = h*w +N = d +gram_matrix_evaluator = K.function([content_model.input],[gram_matrix(fmap,M,N)]) +# print(gram_matrix_evaluator([content_img])[0].mean()) + +def style_layer_loss(Fa, Fx): + #! Change me + _, h, w, d = Fa.get_shape().as_list() + M = h * w + N = d + constant = 1/(4*M**2*N**2) + suum = tf.reduce_sum((gram_matrix(Fa, M, N) - gram_matrix(Fx, M, N))**2) + loss = constant*suum + return loss + +style_loss_0 = style_layer_loss(style_model.output[0],blend_style_outputs[0]) + +# The correct output of this function is 220990.31 +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +content_loss_evaluator = K.function([blend_model.input],[style_loss_0]) +# print(content_loss_evaluator([input_img])) + +style_loss = 0 +for i in range(5): + style_loss += 0.2*style_layer_loss(style_model.output[i],blend_style_outputs[i]) + +# The correct output of this function is 177059700.0 +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +style_loss_evaluator = K.function([blend_model.input],[style_loss]) +# print (style_loss_evaluator([input_img])) + +tv_loss = K.tf.image.total_variation(blend_model.input) + +alpha = 5.0 +beta = 2e3 +gamma = 1e-3 + +total_loss = alpha*content_loss + beta*style_loss + gamma*tv_loss + +# The correct output of this function is 1.7715756e+12 +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +total_loss_evaluator = K.function([blend_model.input],[total_loss]) +# print (total_loss_evaluator([input_img])) + +grads = K.gradients(total_loss,blend_model.input)[0] + +loss_and_grad_evaluator = K.function([blend_model.input],[total_loss,grads]) + +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +l0,g0 = loss_and_grad_evaluator([input_img]) +# Correct value of l0 is 3.5509e11 +# Correct value of first element in g0 is -7.28989e2 + +g = None +def internal_loss_and_grad_evaluator(x): + loss_and_grad_evaluator = K.function([blend_model.input],[total_loss,grads]) + np.random.seed(0) + input_img = np.random.randn(1,img_rows,img_cols,3) + l,g = loss_and_grad_evaluator([input_img]) + return loss_and_grad_evaluator +def gradient(x): + return g + +sio.fmin_l_bfgs_b(internal_loss_and_grad_evaluator, [content_img], fprime=gradient) From 2b504d050301025496de385c73148b02fab6891f Mon Sep 17 00:00:00 2001 From: maxthibeau Date: Thu, 25 Apr 2019 14:24:45 -0600 Subject: [PATCH 2/3] Finished neural style transfer mostly still needs to display image and add mean back to color bands. --- neural_style_transfer.py | 150 ++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 65 deletions(-) diff --git a/neural_style_transfer.py b/neural_style_transfer.py index e54be21..727a10c 100644 --- a/neural_style_transfer.py +++ b/neural_style_transfer.py @@ -10,6 +10,60 @@ import scipy.optimize as sio +def content_layer_loss(Fp, Fx): + #! Change me + _,h,w,d = Fp.get_shape().as_list() + constant = 1/(2*(w*h)**.5*d**.5) + suum = tf.reduce_sum(np.sum((Fx[0,:,:,l] - Fp[0,:,:,l])**2 for l in range(0, d))) + loss = constant*suum + return loss + +def gram_matrix(f, M, N): + # Accepts a (height,width,depth)-sized feature map, + # reshapes to (M,N), then computes the inner product + reshaped_f = kl.Reshape((M, N))(f)[0] + reshaped_f_t = K.transpose(reshaped_f) + gram_matrix = K.dot(reshaped_f_t, reshaped_f) + # !Change me + return gram_matrix + +def style_layer_loss(Fa, Fx): + #! Change me + _, h, w, d = Fa.get_shape().as_list() + M = h * w + N = d + constant = 1/(4*M**2*N**2) + suum = tf.reduce_sum((gram_matrix(Fa, M, N) - gram_matrix(Fx, M, N))**2) + loss = constant*suum + return loss + +def total_loss(model_input): + tv_loss = K.tf.image.total_variation(blend_model.input) + alpha = 5.0 + beta = 2e3 + gamma = 1e-3 + total_loss = alpha*content_loss + beta*style_loss + gamma*tv_loss + return total_loss + +class NeuralStyleTransfer: + + def __init__(self, blend_model_input): + self._g = None + self._blend_model_input = blend_model_input + + def loss_and_grad_eval(self, input_img, width, height): + input_img = input_img.reshape(1, width, height, 3) + total_loss_val = total_loss(self._blend_model_input) + grads = K.gradients(total_loss_val,self._blend_model_input)[0] + + loss_and_grad_evaluator = K.function([self._blend_model_input],[total_loss_val,grads]) + l,self._g = loss_and_grad_evaluator([input_img]) + self._g = self._g.flatten().astype('float64') + return l.astype('float64') + + def gradient(self, input_img, width, height): + return self._g + content_path = './main_hall.jpg' style_path = './starry_night.jpg' @@ -93,55 +147,29 @@ # Separate the model outputs into those intended for comparison with the content layer and the style layer blend_content_outputs = [blend_model.outputs[0]] blend_style_outputs = blend_model.outputs[1:] -def content_layer_loss(Fp, Fx): - #! Change me - _,h,w,d = Fp.get_shape().as_list() - constant = 1/(2*(w*h)**.5*d**.5) - suum = tf.reduce_sum(np.sum((Fx[0,:,:,l] - Fp[0,:,:,l])**2 for l in range(0, d))) - loss = constant*suum - return loss content_loss = content_layer_loss(content_model.output,blend_content_outputs[0]) # The correct output of this function is 195710720.0 -np.random.seed(0) -input_img = np.random.randn(1,img_rows,img_cols,3) -content_loss_evaluator = K.function([blend_model.input],[content_loss]) +# np.random.seed(0) +# input_img = np.random.randn(1,img_rows,img_cols,3) +# content_loss_evaluator = K.function([blend_model.input],[content_loss]) # print(content_loss_evaluator([input_img])) -def gram_matrix(f, M, N): - # Accepts a (height,width,depth)-sized feature map, - # reshapes to (M,N), then computes the inner product - reshaped_f = kl.Reshape((M, N))(f)[0] - reshaped_f_t = K.transpose(reshaped_f) - gram_matrix = K.dot(reshaped_f_t, reshaped_f) - # !Change me - return gram_matrix - # For a correctly implemented gram_matrix, the following code will produce 113934860.0 -fmap = content_model.output -_,h,w,d = fmap.get_shape() -M = h*w -N = d -gram_matrix_evaluator = K.function([content_model.input],[gram_matrix(fmap,M,N)]) +# fmap = content_model.output +# _,h,w,d = fmap.get_shape() +# M = h*w +# N = d +# gram_matrix_evaluator = K.function([content_model.input],[gram_matrix(fmap,M,N)]) # print(gram_matrix_evaluator([content_img])[0].mean()) -def style_layer_loss(Fa, Fx): - #! Change me - _, h, w, d = Fa.get_shape().as_list() - M = h * w - N = d - constant = 1/(4*M**2*N**2) - suum = tf.reduce_sum((gram_matrix(Fa, M, N) - gram_matrix(Fx, M, N))**2) - loss = constant*suum - return loss - -style_loss_0 = style_layer_loss(style_model.output[0],blend_style_outputs[0]) +# style_loss_0 = style_layer_loss(style_model.output[0],blend_style_outputs[0]) # The correct output of this function is 220990.31 -np.random.seed(0) -input_img = np.random.randn(1,img_rows,img_cols,3) -content_loss_evaluator = K.function([blend_model.input],[style_loss_0]) +# np.random.seed(0) +# input_img = np.random.randn(1,img_rows,img_cols,3) +# content_loss_evaluator = K.function([blend_model.input],[style_loss_0]) # print(content_loss_evaluator([input_img])) style_loss = 0 @@ -149,43 +177,35 @@ def style_layer_loss(Fa, Fx): style_loss += 0.2*style_layer_loss(style_model.output[i],blend_style_outputs[i]) # The correct output of this function is 177059700.0 -np.random.seed(0) -input_img = np.random.randn(1,img_rows,img_cols,3) -style_loss_evaluator = K.function([blend_model.input],[style_loss]) +# np.random.seed(0) +# input_img = np.random.randn(1,img_rows,img_cols,3) +# style_loss_evaluator = K.function([blend_model.input],[style_loss]) # print (style_loss_evaluator([input_img])) -tv_loss = K.tf.image.total_variation(blend_model.input) +# tv_loss = K.tf.image.total_variation(blend_model.input) -alpha = 5.0 -beta = 2e3 -gamma = 1e-3 +# alpha = 5.0 +# beta = 2e3 +# gamma = 1e-3 -total_loss = alpha*content_loss + beta*style_loss + gamma*tv_loss +# total_loss = alpha*content_loss + beta*style_loss + gamma*tv_loss # The correct output of this function is 1.7715756e+12 -np.random.seed(0) -input_img = np.random.randn(1,img_rows,img_cols,3) -total_loss_evaluator = K.function([blend_model.input],[total_loss]) +# np.random.seed(0) +# input_img = np.random.randn(1,img_rows,img_cols,3) +# total_loss_evaluator = K.function([blend_model.input],[total_loss]) # print (total_loss_evaluator([input_img])) -grads = K.gradients(total_loss,blend_model.input)[0] +# grads = K.gradients(total_loss,blend_model.input)[0] -loss_and_grad_evaluator = K.function([blend_model.input],[total_loss,grads]) +# loss_and_grad_evaluator = K.function([blend_model.input],[total_loss,grads]) -np.random.seed(0) -input_img = np.random.randn(1,img_rows,img_cols,3) -l0,g0 = loss_and_grad_evaluator([input_img]) +# np.random.seed(0) +# input_img = np.random.randn(1,img_rows,img_cols,3) +# l0,g0 = loss_and_grad_evaluator([input_img]) # Correct value of l0 is 3.5509e11 # Correct value of first element in g0 is -7.28989e2 -g = None -def internal_loss_and_grad_evaluator(x): - loss_and_grad_evaluator = K.function([blend_model.input],[total_loss,grads]) - np.random.seed(0) - input_img = np.random.randn(1,img_rows,img_cols,3) - l,g = loss_and_grad_evaluator([input_img]) - return loss_and_grad_evaluator -def gradient(x): - return g - -sio.fmin_l_bfgs_b(internal_loss_and_grad_evaluator, [content_img], fprime=gradient) +neddy = NeuralStyleTransfer(blend_model.input) + +sio.fmin_l_bfgs_b(neddy.loss_and_grad_eval, content_img.flatten(), fprime=neddy.gradient, args=(content_img.shape[1], content_img.shape[2]), iprint=10) From da11a7cd1ac7289853611d4095ddfcc5a1947db3 Mon Sep 17 00:00:00 2001 From: maxthibeau Date: Sat, 27 Apr 2019 18:37:05 -0600 Subject: [PATCH 3/3] got it completely working :) --- neural_style_transfer.py | 150 +++++++++++++-------------------------- 1 file changed, 51 insertions(+), 99 deletions(-) diff --git a/neural_style_transfer.py b/neural_style_transfer.py index 727a10c..7608008 100644 --- a/neural_style_transfer.py +++ b/neural_style_transfer.py @@ -8,62 +8,9 @@ import keras.models as km import tensorflow as tf +import scipy.misc import scipy.optimize as sio -def content_layer_loss(Fp, Fx): - #! Change me - _,h,w,d = Fp.get_shape().as_list() - constant = 1/(2*(w*h)**.5*d**.5) - suum = tf.reduce_sum(np.sum((Fx[0,:,:,l] - Fp[0,:,:,l])**2 for l in range(0, d))) - loss = constant*suum - return loss - -def gram_matrix(f, M, N): - # Accepts a (height,width,depth)-sized feature map, - # reshapes to (M,N), then computes the inner product - reshaped_f = kl.Reshape((M, N))(f)[0] - reshaped_f_t = K.transpose(reshaped_f) - gram_matrix = K.dot(reshaped_f_t, reshaped_f) - # !Change me - return gram_matrix - -def style_layer_loss(Fa, Fx): - #! Change me - _, h, w, d = Fa.get_shape().as_list() - M = h * w - N = d - constant = 1/(4*M**2*N**2) - suum = tf.reduce_sum((gram_matrix(Fa, M, N) - gram_matrix(Fx, M, N))**2) - loss = constant*suum - return loss - -def total_loss(model_input): - tv_loss = K.tf.image.total_variation(blend_model.input) - alpha = 5.0 - beta = 2e3 - gamma = 1e-3 - total_loss = alpha*content_loss + beta*style_loss + gamma*tv_loss - return total_loss - -class NeuralStyleTransfer: - - def __init__(self, blend_model_input): - self._g = None - self._blend_model_input = blend_model_input - - def loss_and_grad_eval(self, input_img, width, height): - input_img = input_img.reshape(1, width, height, 3) - total_loss_val = total_loss(self._blend_model_input) - grads = K.gradients(total_loss_val,self._blend_model_input)[0] - - loss_and_grad_evaluator = K.function([self._blend_model_input],[total_loss_val,grads]) - l,self._g = loss_and_grad_evaluator([input_img]) - self._g = self._g.flatten().astype('float64') - return l.astype('float64') - - def gradient(self, input_img, width, height): - return self._g - content_path = './main_hall.jpg' style_path = './starry_night.jpg' @@ -148,64 +95,69 @@ def gradient(self, input_img, width, height): blend_content_outputs = [blend_model.outputs[0]] blend_style_outputs = blend_model.outputs[1:] -content_loss = content_layer_loss(content_model.output,blend_content_outputs[0]) - -# The correct output of this function is 195710720.0 -# np.random.seed(0) -# input_img = np.random.randn(1,img_rows,img_cols,3) -# content_loss_evaluator = K.function([blend_model.input],[content_loss]) -# print(content_loss_evaluator([input_img])) +def content_layer_loss(Fp, Fx): + #! Change me + _,h,w,d = Fp.get_shape().as_list() + constant = 1/(2*(w*h)**.5*d**.5) + suum = tf.reduce_sum(np.sum((Fx[0,:,:,l] - Fp[0,:,:,l])**2 for l in range(0, d))) + loss = constant*suum + return loss -# For a correctly implemented gram_matrix, the following code will produce 113934860.0 -# fmap = content_model.output -# _,h,w,d = fmap.get_shape() -# M = h*w -# N = d -# gram_matrix_evaluator = K.function([content_model.input],[gram_matrix(fmap,M,N)]) -# print(gram_matrix_evaluator([content_img])[0].mean()) +content_loss = content_layer_loss(content_model.output,blend_content_outputs[0]) -# style_loss_0 = style_layer_loss(style_model.output[0],blend_style_outputs[0]) +def gram_matrix(f, M, N): + # Accepts a (height,width,depth)-sized feature map, + # reshapes to (M,N), then computes the inner product + reshaped_f = kl.Reshape((M, N))(f)[0] + reshaped_f_t = K.transpose(reshaped_f) + gram_matrix = K.dot(reshaped_f_t, reshaped_f) + # !Change me + return gram_matrix -# The correct output of this function is 220990.31 -# np.random.seed(0) -# input_img = np.random.randn(1,img_rows,img_cols,3) -# content_loss_evaluator = K.function([blend_model.input],[style_loss_0]) -# print(content_loss_evaluator([input_img])) +def style_layer_loss(Fa, Fx): + #! Change me + _, h, w, d = Fa.get_shape().as_list() + M = h * w + N = d + constant = 1/(4*M**2*N**2) + suum = tf.reduce_sum((gram_matrix(Fa, M, N) - gram_matrix(Fx, M, N))**2) + loss = constant*suum + return loss style_loss = 0 for i in range(5): style_loss += 0.2*style_layer_loss(style_model.output[i],blend_style_outputs[i]) -# The correct output of this function is 177059700.0 -# np.random.seed(0) -# input_img = np.random.randn(1,img_rows,img_cols,3) -# style_loss_evaluator = K.function([blend_model.input],[style_loss]) -# print (style_loss_evaluator([input_img])) +tv_loss = K.tf.image.total_variation(blend_model.input) +alpha = 5.0 +beta = 2e3 +gamma = 1e-3 +total_loss = alpha*content_loss + beta*style_loss + gamma*tv_loss -# tv_loss = K.tf.image.total_variation(blend_model.input) +grads = K.gradients(total_loss,blend_model.input)[0] -# alpha = 5.0 -# beta = 2e3 -# gamma = 1e-3 +loss_and_grad_evaluator = K.function([blend_model.input],[total_loss,grads]) -# total_loss = alpha*content_loss + beta*style_loss + gamma*tv_loss - -# The correct output of this function is 1.7715756e+12 -# np.random.seed(0) -# input_img = np.random.randn(1,img_rows,img_cols,3) -# total_loss_evaluator = K.function([blend_model.input],[total_loss]) -# print (total_loss_evaluator([input_img])) +global g -# grads = K.gradients(total_loss,blend_model.input)[0] +def loss_and_grad_eval(input_img, width, height): + global g + input_img = input_img.reshape(1, width, height, 3) -# loss_and_grad_evaluator = K.function([blend_model.input],[total_loss,grads]) + l, g = loss_and_grad_evaluator([input_img]) + g = g.flatten().astype('float64') + return l.astype('float64') -# np.random.seed(0) -# input_img = np.random.randn(1,img_rows,img_cols,3) -# l0,g0 = loss_and_grad_evaluator([input_img]) -# Correct value of l0 is 3.5509e11 -# Correct value of first element in g0 is -7.28989e2 +def gradient(input_img, width, height): + global g + return g -neddy = NeuralStyleTransfer(blend_model.input) +output_img, f, d = sio.fmin_l_bfgs_b(loss_and_grad_eval, content_img.flatten(), fprime=gradient, args=(content_img.shape[1], content_img.shape[2]), iprint=10, maxfun=500) -sio.fmin_l_bfgs_b(neddy.loss_and_grad_eval, content_img.flatten(), fprime=neddy.gradient, args=(content_img.shape[1], content_img.shape[2]), iprint=10) +output_img = np.reshape(output_img, (img_rows, img_cols, 3)) +output_img[:, :, 0] += 103.939 +output_img[:, :, 1] += 116.779 +output_img[:, :, 2] += 123.68 +#plt.imshow(output_img.astype(int)) +#plt.show() +scipy.misc.imsave('outfile.jpg', output_img)