diff --git a/neural_style_transfer.ipynb b/neural_style_transfer.ipynb index 2ce13a7..bc414da 100644 --- a/neural_style_transfer.ipynb +++ b/neural_style_transfer.ipynb @@ -25,13 +25,40 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + }, + { + "data": { + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import keras.preprocessing as kp\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", + "import keras.backend as K\n", "\n", "content_path = './main_hall.jpg'\n", "style_path = './starry_night.jpg'\n", @@ -79,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -103,9 +130,31 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /home/dunbar/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Colocations handled automatically by placer.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUcAAAD8CAYAAADkM2ZpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAD1dJREFUeJzt3W1snfV9xvHr8nEcx04yEpJQEkNDEUubMWiQxUOZqomAFCgifdFqoFGxjilv1pZWnTpQpfXdhLSqa6VVnSJKi9QINKV0RVUfyEI7VI2yhkBDQmBJyZMhJCHhISQEx/ZvL3w6BfMn9nz/cu4b9P1IkX2Ob/3OFfv48v/c5z73cUQIAPB2XXUHAIAmohwBoIByBIACyhEACihHACigHAGggHIEgALKEQAKKEcAKOju5I31eGb0qr+TN/ne4sRRM2YkDUoKdXIkZUyMjaXMwXtQ0l3xaLzyckQsnGy7jpZjr/p1hVd28ibfU9yd9+NoLT43aVArZczYgUM5c44dS5mTVvq8/LZjsn4/Npx8YM9UtuNhNQAUUI4AUEA5AkAB5QgABZXK0fYq28/Z3mn7zqxQAFC3aZej7Zakb0u6XtJySbfYXp4VDADqVGXleLmknRHxfEQMS3pA0uqcWABQryrluETSvlMuD7Wvexvba2xvsr3ppN6qcHMA0DlVyrF0FO07joiNiLURMRgRgzM0s8LNAUDnVCnHIUnnnXJ5QNKL1eIAQDNUKcffSrrI9gW2eyTdLOmhnFgAUK9pv1gxIkZsf07SLyS1JN0bEdvSkgFAjSq9kjsifirpp0lZAKAxeIUMABRQjgBQQDkCQEFHT3abxTObdbxk1/nvOPZ9Wl6/dNKTE0/ZkWU5J6nN+vM5Z0/OyXfn7jqRMqd1IufM5F3Hh1PmxK59k280lTmjoylzJCmSzt7evfgDKXNGXzqQMmeqWDkCQAHlCAAFlCMAFFCOAFBAOQJAAeUIAAWUIwAUUI4AUEA5AkAB5QgABZQjABRQjgBQQDkCQAHlCAAFlCMAFFCOAFBAOQJAQUfPBG5bXb29lefERy5MSCN17d2fMkcHXk4ZM/v3s1LmSNKxRX+UMufEgpQxGp7rlDndr72ZMmdsy7Mpc/LOu50j4/fr/2Zd+MGUOZEyRYqRnDOTTxUrRwAooBwBoIByBIACyhEACihHACiYdjnaPs/2L21vt73N9h2ZwQCgTlUO5RmR9OWI2Gx7jqQnbG+IiGeSsgFAbaa9coyI/RGxuf35UUnbJS3JCgYAdUrZ52h7qaQVkh7PmAcAdav8ChnbsyX9UNIXI+L1wtfXSFojSb3ur3pzANARlVaOtmdovBjXRcSDpW0iYm1EDEbEYI9mVrk5AOiYKs9WW9J3JW2PiG/kRQKA+lVZOV4t6TOSrrH9VPvfDUm5AKBW097nGBG/lpRzqhUAaBheIQMABZQjABRQjgBQ0NEzgY/M69MrN3y08pw3F+Z0+sC/H02ZM7JnX8qcsbk9KXMk6dU/zTlrct+iYylzjj8zN2XOWO+MlDnvV4f/YkXarFeW58xZ8FTOucDn7ng+Zc5UsXIEgALKEQAKKEcAKKAcAaCAcgSAAsoRAAooRwAooBwBoIByBIACyhEACihHACigHAGggHIEgALKEQAKKEcAKKAcAaCAcgSAAsoRAAo6+jYJ3W+Oat7W1yrPOfv14wlppJHde1PmZOl6azRtVt/eWSlzjrf6Uub0JL2J7xtL+1PmzP7vlDGNc3Rp3rslf/b6jSlz1r26MmVOzhttTB0rRwAooBwBoIByBIACyhEACiqXo+2W7Sdt/yQjEAA0QcbK8Q5J2xPmAEBjVCpH2wOSPiHpnpw4ANAMVVeO35T0FUljCVkAoDGmXY62b5R0MCKemGS7NbY32d40PJJz8DYAnGlVVo5XS7rJ9m5JD0i6xvYPJm4UEWsjYjAiBnu6c15tAQBn2rTLMSLuioiBiFgq6WZJj0TErWnJAKBGHOcIAAUpJ56IiF9J+lXGLABoAlaOAFBAOQJAAeUIAAWUIwAUdPRM4PHmCY1tebbynLGIhDTN09oxlDar+4oPp8z580uq/7wk6fr5W1LmbLnx/JQ5jx2+PGVO98bTvgai45Zd8/u0WdfO2ZoyZ+Pjf5Yyp9NYOQJAAeUIAAWUIwAUUI4AUEA5AkAB5QgABZQjABRQjgBQQDkCQAHlCAAFlCMAFFCOAFBAOQJAAeUIAAWUIwAUUI4AUEA5AkBBR88ELkl6n57FO8Po4SNps87e9lbKnL9Z9J8pc67uzfk73ONdKXPWffqqlDl/vDFljLp6e1Pm7PzZhSlzJOn2sTtS5ix+5L9S5nQaK0cAKKAcAaCAcgSAAsoRAAoqlaPts2yvt/2s7e22c/ZyA0DNqj5b/S1JP4+IT9nukdSXkAkAajftcrQ9V9LHJf2VJEXEsKThnFgAUK8qD6s/JOmQpO/ZftL2Pbb7k3IBQK2qlGO3pMskfSciVkg6JunOiRvZXmN7k+1NJ5VzYDIAnGlVynFI0lBEPN6+vF7jZfk2EbE2IgYjYnCGZla4OQDonGmXY0S8JGmf7WXtq1ZKeiYlFQDUrOqz1Z+XtK79TPXzkj5bPRIA1K9SOUbEU5IGk7IAQGPwChkAKKAcAaCAcgSAAsoRAAocHTwz91zPjyu8smO31zF2ypju8wdS5kjSG5ecmzJndGbO/81jKWPUv+eNlDmtFw+nzBnZ/1LKHHTOf8T6JyJi0ieSWTkCQAHlCAAFlCMAFFCOAFBAOQJAAeUIAAWUIwAUUI4AUEA5AkAB5QgABZQjABRQjgBQQDkCQAHlCAAFlCMAFFCOAFBAOQJAQdX3rf7/Szhrdmv+vIQg0ujhIylzskTPjLRZszcPpcwZeeHFlDlZss5bP5I0p7VwYcoc989KmTOye2/KnEytP1mWMmd023Mpc6aKlSMAFFCOAFBAOQJAAeUIAAWUIwAUVCpH21+yvc32Vtv32+7NCgYAdZp2OdpeIukLkgYj4mJJLUk3ZwUDgDpVfVjdLWmW7W5JfZKadVAcAEzTtMsxIl6Q9HVJeyXtl/RaRDw8cTvba2xvsr3ppN6aflIA6KAqD6vnSVot6QJJiyX127514nYRsTYiBiNicIZmTj8pAHRQlYfV10raFRGHIuKkpAclfSwnFgDUq0o57pV0pe0+25a0UtL2nFgAUK8q+xwfl7Re0mZJT7dnrU3KBQC1qnRWnoj4mqSvJWUBgMbgFTIAUEA5AkAB5QgABZ0/E3hUP5dzDJ9MCJIo4f8kSTryas4cSWMnmnXAfWvB2SlzRl8+nDInSxw9mjPn+PGUOU104tzZKXNm7p2TMkevT20zVo4AUEA5AkAB5QgABZQjABRQjgBQQDkCQAHlCAAFlCMAFFCOAFBAOQJAAeUIAAWUIwAUUI4AUEA5AkAB5QgABZQjABRQjgBQQDkCQEFH3ybBXV3q6uuvPCeGhxPSNM/o4SN1Rzhjxl7LeTuB1tnzU+Zkfa/HTpxImRNXXZoyp3Us7+0xvPvFlDldv96WMifrez1VrBwBoIByBIACyhEACihHACiYtBxt32v7oO2tp1w33/YG2zvaH+ed2ZgA0FlTWTl+X9KqCdfdKWljRFwkaWP7MgC8b0xajhHxqKSJxz2slnRf+/P7JH0yORcA1Gq6+xzPiYj9ktT+uCgvEgDU74wfBG57jaQ1ktTr6geAA0AnTHfleMD2uZLU/njw3TaMiLURMRgRgz3unebNAUBnTbccH5J0W/vz2yT9OCcOADTDVA7luV/SY5KW2R6yfbukuyVdZ3uHpOvalwHgfWPSfY4Rccu7fGllchYAaAxeIQMABZQjABRQjgBQQDkCQEFHzwSuGd3qOmdh5TEji+YmhEn0my11J3iHrv6cA+49O+nA/eGTOXPmn5UypjU6mjIny7GzZ6bM6Wk5ZY4kdZ+/OGXOiYE5KXN6H805o7iOTW0zVo4AUEA5AkAB5QgABZQjABRQjgBQQDkCQAHlCAAFlCMAFFCOAFBAOQJAAeUIAAWUIwAUUI4AUEA5AkAB5QgABZQjABRQjgBQ4Ijo3I3ZhyTtmWSzBZJe7kCcqSLP5JqWiTyn17Q8UmczfTAiJn1Lgo6W41TY3hQRg3Xn+APyTK5pmchzek3LIzUzEw+rAaCAcgSAgiaW49q6A0xAnsk1LRN5Tq9peaQGZmrcPkcAaIImrhwBoHaNKUfbq2w/Z3un7TsbkOc827+0vd32Ntt31J1Jkmy3bD9p+ycNyHKW7fW2n21/n66qOc+X2j+rrbbvt91bQ4Z7bR+0vfWU6+bb3mB7R/vjvJrz/FP7Z7bF9o9sn1VnnlO+9ne2w/aCTuU5nUaUo+2WpG9Lul7Sckm32F5ebyqNSPpyRHxE0pWS/rYBmSTpDknb6w7R9i1JP4+ID0u6VDXmsr1E0hckDUbExZJakm6uIcr3Ja2acN2dkjZGxEWSNrYv15lng6SLI+ISSf8j6a6a88j2eZKuk7S3g1lOqxHlKOlySTsj4vmIGJb0gKTVdQaKiP0Rsbn9+VGN/+IvqTOT7QFJn5B0T5052lnmSvq4pO9KUkQMR8Sr9aZSt6RZtrsl9Ul6sdMBIuJRSUcmXL1a0n3tz++T9Mk680TEwxEx0r74G0kDdeZp+2dJX5HUmCdBmlKOSyTtO+XykGouolPZXipphaTH602ib2r8DjRWcw5J+pCkQ5K+136Yf4/t/rrCRMQLkr6u8ZXHfkmvRcTDdeWZ4JyI2C+N/9GVtKjmPKf6a0k/qzOA7ZskvRARv6szx0RNKUcXrmvEXxDbsyX9UNIXI+L1GnPcKOlgRDxRV4YJuiVdJuk7EbFC0jF19uHi27T3462WdIGkxZL6bd9aV573Attf1fjuo3U1ZuiT9FVJ/1BXhnfTlHIcknTeKZcHVMNDoolsz9B4Ma6LiAdrjnO1pJts79b4bodrbP+gxjxDkoYi4g+r6fUaL8u6XCtpV0QcioiTkh6U9LEa85zqgO1zJan98WDNeWT7Nkk3SvrLqPd4vgs1/gftd+379oCkzbY/UGMmSc0px99Kusj2BbZ7NL4j/aE6A9m2xvenbY+Ib9SZRZIi4q6IGIiIpRr//jwSEbWtjCLiJUn7bC9rX7VS0jN15dH4w+krbfe1f3Yr1Zwnrh6SdFv789sk/bjGLLK9StLfS7opIo7XmSUino6IRRGxtH3fHpJ0Wfv+VatGlGN75/DnJP1C43fof4uIbfWm0tWSPqPxFdpT7X831JypaT4vaZ3tLZI+Kukf6wrSXsGul7RZ0tMav293/FUXtu+X9JikZbaHbN8u6W5J19neofFnZO+uOc+/SJojaUP7fv2vNedpJF4hAwAFjVg5AkDTUI4AUEA5AkAB5QgABZQjABRQjgBQQDkCQAHlCAAF/wufap9bY7P+3AAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "import vgg\n", "import keras.backend as K\n", @@ -119,7 +168,107 @@ "evaluator = K.function([content_base_model.input],[content_base_model.output])\n", "feature_maps = evaluator([content_img])\n", "plt.imshow(feature_maps[0][0,:,:,500])\n", - "plt.show()" + "plt.show()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "input_1 (InputLayer) (1, 400, 543, 3) 0 \n", + "_________________________________________________________________\n", + "block1_conv1 (Conv2D) (1, 400, 543, 64) 1792 \n", + "_________________________________________________________________\n", + "block1_relu1 (Activation) (1, 400, 543, 64) 0 \n", + "_________________________________________________________________\n", + "block1_conv2 (Conv2D) (1, 400, 543, 64) 36928 \n", + "_________________________________________________________________\n", + "block1_relu2 (Activation) (1, 400, 543, 64) 0 \n", + "_________________________________________________________________\n", + "block1_pool (AveragePooling2 (1, 200, 271, 64) 0 \n", + "_________________________________________________________________\n", + "block2_conv1 (Conv2D) (1, 200, 271, 128) 73856 \n", + "_________________________________________________________________\n", + "block2_relu1 (Activation) (1, 200, 271, 128) 0 \n", + "_________________________________________________________________\n", + "block2_conv2 (Conv2D) (1, 200, 271, 128) 147584 \n", + "_________________________________________________________________\n", + "block2_relu2 (Activation) (1, 200, 271, 128) 0 \n", + "_________________________________________________________________\n", + "block2_pool (AveragePooling2 (1, 100, 135, 128) 0 \n", + "_________________________________________________________________\n", + "block3_conv1 (Conv2D) (1, 100, 135, 256) 295168 \n", + "_________________________________________________________________\n", + "block3_relu1 (Activation) (1, 100, 135, 256) 0 \n", + "_________________________________________________________________\n", + "block3_conv2 (Conv2D) (1, 100, 135, 256) 590080 \n", + "_________________________________________________________________\n", + "block3_relu2 (Activation) (1, 100, 135, 256) 0 \n", + "_________________________________________________________________\n", + "block3_conv3 (Conv2D) (1, 100, 135, 256) 590080 \n", + "_________________________________________________________________\n", + "block3_relu3 (Activation) (1, 100, 135, 256) 0 \n", + "_________________________________________________________________\n", + "block3_conv4 (Conv2D) (1, 100, 135, 256) 590080 \n", + "_________________________________________________________________\n", + "block3_relu4 (Activation) (1, 100, 135, 256) 0 \n", + "_________________________________________________________________\n", + "block3_pool (AveragePooling2 (1, 50, 67, 256) 0 \n", + "_________________________________________________________________\n", + "block4_conv1 (Conv2D) (1, 50, 67, 512) 1180160 \n", + "_________________________________________________________________\n", + "block4_relu1 (Activation) (1, 50, 67, 512) 0 \n", + "_________________________________________________________________\n", + "block4_conv2 (Conv2D) (1, 50, 67, 512) 2359808 \n", + "_________________________________________________________________\n", + "block4_relu2 (Activation) (1, 50, 67, 512) 0 \n", + "_________________________________________________________________\n", + "block4_conv3 (Conv2D) (1, 50, 67, 512) 2359808 \n", + "_________________________________________________________________\n", + "block4_relu3 (Activation) (1, 50, 67, 512) 0 \n", + "_________________________________________________________________\n", + "block4_conv4 (Conv2D) (1, 50, 67, 512) 2359808 \n", + "_________________________________________________________________\n", + "block4_relu4 (Activation) (1, 50, 67, 512) 0 \n", + "_________________________________________________________________\n", + "block4_pool (AveragePooling2 (1, 25, 33, 512) 0 \n", + "_________________________________________________________________\n", + "block5_conv1 (Conv2D) (1, 25, 33, 512) 2359808 \n", + "_________________________________________________________________\n", + "block5_relu1 (Activation) (1, 25, 33, 512) 0 \n", + "_________________________________________________________________\n", + "block5_conv2 (Conv2D) (1, 25, 33, 512) 2359808 \n", + "_________________________________________________________________\n", + "block5_relu2 (Activation) (1, 25, 33, 512) 0 \n", + "_________________________________________________________________\n", + "block5_conv3 (Conv2D) (1, 25, 33, 512) 2359808 \n", + "_________________________________________________________________\n", + "block5_relu3 (Activation) (1, 25, 33, 512) 0 \n", + "_________________________________________________________________\n", + "block5_conv4 (Conv2D) (1, 25, 33, 512) 2359808 \n", + "_________________________________________________________________\n", + "block5_relu4 (Activation) (1, 25, 33, 512) 0 \n", + "_________________________________________________________________\n", + "block5_pool (AveragePooling2 (1, 12, 16, 512) 0 \n", + "=================================================================\n", + "Total params: 20,024,384\n", + "Trainable params: 20,024,384\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "content_base_model.summary()" ] }, { @@ -131,7 +280,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -148,9 +297,22 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUgAAAD8CAYAAAAVOD3kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnWuMZVl13//rnvuud3V19/R0z5OMMaAYsCcYhxA5YCLsWLYVOZEdKyISEl8cCSuObEikKJYSyf5i4w+RI2Qc88Ey+JVAELKDCCRyYgPNy8wwDDMMw3RPd09VP+p134+dD3Wbuuu/1ul7prrndjVZP6nVtU+dffY++5yz763/Wue/JaWEIAiCwFK62x0IgiA4rsQEGQRBkENMkEEQBDnEBBkEQZBDTJBBEAQ5xAQZBEGQQ0yQQRAEOcQEGQRBkMNtTZAi8k4ReVpEnhWR992pTgVBEBwH5Khv0ohIBuCbAN4B4CKALwD4+ZTS1/PqZIsLqby+fniMsd1nbu/1yN1qOCgCXx6PuGTByyI7vGOG125gtN+aeZuVb6O5NwF4NqX0HACIyEcA/DSA3AmyvL6OM7/y3sPyvv0Cm3hTkSflKBxlgowncm4ceYKcVfGVuoZ34j71+vZK3f+z2j5Ku0cZ2zt1fgX6P1wZfffnK//xtwsd9nb+xD4L4MJU+eJkWxAEwfcEtzNBenO/+QwRkfeIyHkROT/a37+N5oIgCObL7fyJfRHAA1PlcwAu8U4ppQ8C+CAA1B54IJUGh/Nq1nXm2DuhDb5SX9uLND2jTirQt1nHmCfc3yJ9O0qdo+CO5XHRlu+miDqvP8uZY3Q+RqoDMFqYPlCxg97ON8gvAHhMRB4RkSqAnwPw8ds4XhAEwbHiyN8gU0pDEfmXAP4CQAbg91JKT96xngVBENxlbudPbKSUPgngk3eoL0EQBMeKeJMmCIIgh9v6BnkUigQppjlKUMATi49L4ONu9uPljv0r2e6dGAfvGEeK6b0C16TIIe9Uu0Xu/5fNMc/UT1misv79mMoAkGpTb6YUHPz4BhkEQZBDTJBBEAQ5xAQZBEGQw9w1yGmDCs+s4iiJvkYu8eq8EnrJndJp+DhULpQQXaTtGe14dcw1KjC2Qh+7XtKu6RpvOMq4eceZFzwGc+yHaavIM3SUd9bv0DVSh/DujbI+8Lhq9xnVaJ863ahOu9Xl3uGvs9AggyAIbouYIIMgCHKICTIIgiCHuWuQirtoRHEUjehIeYQFdL9Z5g6usTDneR3lhX6nTmlE5QGV+7YO93dcobJzlxntqcj1uAMf54VyMnm8PZ2Mr5nZ4eX167aYoVkXkqsLjK0bM3i5FNHBmSK690AfSMb2wOXy4c0tkQcZBEFwe8QEGQRBkENMkEEQBDnEBBkEQZDD/IM0UwqyazZwJ168L2JWMa9FigokfZugzFCXsx4MHKRxRXZqi4Mn/MI/AMhIV8q6+vfljq1j29bHcM+5SAI6YYI9RYInRRK4OceYym6eNbddJIAx6x4rtIhJgWaoL6Whv980PLZFgjaFmPGceYGfEgVcym27z7Cpy/01ShxftCe91Dh8kEqlCNIEQRDcFjFBBkEQ5BATZBAEQQ7HLlF8ZjLqERNwjQ5WRHM8wgv9s5KOi+hB3NdRzdmHPtoyJ4GbdaVRdbYJwLjCRqS6M6OaHZRZ5+z1jXVWo/t5q9I1aJ8j6Lmsw3p1+LheX/i4fF1dE4YCJh6mnQJabZrxFMvI2TZrvIvooXcgPuDp67xtVLf7DBb1gcobWiyv1ekNBwAL1cMbsRSJ4kEQBLdHTJBBEAQ5xAQZBEGQQ0yQQRAEOczfUTz5P9/kjpigHCUAc4fcwVngZ3ccN5hCSd+jRV0eLDqRK064HdkTGFNQJpkAzOwo03BJf4ZmLfuZyg4/5TYlmzvnzInuwwVd9oIpHHRyBX5KKvYCFAzvw0EPz42IgzKVPRrbkr0eHGQaUZDMDbbQJeL7CQCG/AJAhSNVTmCNjsPn6D2bR3HzMYn7nJTvnA+PS+eUbVju00GZxQVdTk4Eb6lyeMNkBU8mvkEGQRDkEBNkEARBDjFBBkEQ5DBfDTJBaypHWW2taDszDmt2uUMGF7OSgYdO0isnwg5JcxzXbOcSv2xf5KOO61Rm6zAmB7w6e4nFIdVifREARnXWQ/XvSz3bDu8zdhLoWfQySeDOOLFrumnHSahnXZX1UHN94CTdswbpXUNO9nf24eR+6+jumZI4bU1RGjrXmfXDAonjvI9xqHdenOitUrMbNun7+89sqvJrlq+o8pM7Z0ydRaVBRqJ4EATBbRETZBAEQQ4xQQZBEORwd80qHGZJA0daWbBAO+4L/byNTQx4ZUFgpk7J2htgc9BKfdLRnBzHWfmW3j62/85gO6vB6Yac/lO+5YiMBFKzgENHjwfTfnYn0tJGTv9HNV2PzVeLmPeae8NKYOaaGP3QM7hgybeAKYanHzLufTjjuAxrgUfJefT0RNYc2XDZMz/pr+rGT2zsmX1ON/S2+2o7ut1VO26LU0LxZ7ljOcQ3yCAIghxiggyCIMhh5gQpIr8nIpsi8sTUtnUR+ZSIPDP5f+2V7WYQBMH8KfIN8vcBvJO2vQ/Ap1NKjwH49KQcBEHwPcXMIE1K6X+LyMO0+acB/Ojk5w8D+CyAXy3U4rRZhSMEzxSUjxikmeko7hlnUF/MqoBuAq4+MBs5eAEXUKIvJ4p71K5pZd4LMg0WONqgixwM8rZxIIeTnQFgRInsqaY7Uyrb8xm3b+2OMF62in+lqYX1UmaP22/r7OthhzPQnWtW1/1NFDCqXLWPCQdyBkt03b0YwKwVLl2HCKpTIPjDgSnvOZvpVF4k6ZsuUbll62S9W7vYc1I4AIxX9IFrZXsvXGw5FacYsAMMgAcb17/7cyo4kRxVgzydUroMAJP/Tx3xOEEQBMeWVzxIIyLvEZHzInJ+3HI+YoIgCI4pR50gXxKRMwAw+X8zb8eU0gdTSo+nlB4vLTgv5QZBEBxTjpoo/nEA7wLw65P/P3anOjTzHXJPnpul7XgUeFfdvPTPqwI6pq6lAe3DZgNNJ7l54daaY3nH6ilsUMDJ2R6sL7KxLeAYqVIi79DRXY0WO9Sfu2Pngshwxmezk3Q8YEnVMdtIrPFW9T5eHaGbjs8wc7RaPqX+mq6VdWydyj4dl3RKKXDjFlnJkU+giC9DoWemgB7KsKnHYFkfpL9ur8fCWkeVl2tds89uTzu8fO7iQ6rc2WqaOl85ffgX7Hbvr/wOE0XSfP4QwF8BeLWIXBSRd+NgYnyHiDwD4B2TchAEwfcURaLYP5/zq7ff4b4EQRAcK+JNmiAIghzmalYhIBmjiGflEfREt8osowBnJGZpjp4Gw8cx2o5jBsEaozFwddoZUo6jm5PJRg2shzYc4wmzgJgeOK8d66rLZUeDZEOO4ezcvTGfDy9OBQBsYMEapGdkS7pltqMHqnnJMyzW5Y5JdLN1Glu6XCZpzVscbFSXmftwXwp6wWoKfFUyUifVGSzZOqOG7n/3BD1TCzaBt1TS16zknNBGQ2fEjOhB67atyzEftwjxDTIIgiCHmCCDIAhyiAkyCIIgh5gggyAIcjh2juJ3hAKrJRZK4GYTBg5QeIbc3CwFKLzV+rhvgyVqd8FmTZfJpXvsmGCMWzqqxEYT7gqF7K7NwZSBkzTNhhY8Tl4zdQqecCDHMd9gE4ZS336+j+ocVOLO2uNy/6rbekPzqu3MoMlJ97O/a5Q7um/NLX0N+8v2hYDWfXqbF6SZGZQ5yuqhBeqMaFXJofMM8b0xXtXnvHqCsucBPLJ2TZcXrpl9Fst6GckBjd2zSydNnfpUZv7lSt/83iO+QQZBEOQQE2QQBEEOMUEGQRDkcPw0yJmmokc7LCe18kpwnuFs1p3RGacvs3RLz+CC90mko5VqtnNCSa/iLG0nDdIuOenYW8GQNUY+Z28lRBbBWA/1nBAoYTtRWZzseHMYpy+zjDMSG+gCqN7Q+zSu6mO0Ttm+DEmD5OR+TuQHgL0H9XHqN3icTBVDEYMIg/fMzDCe8JPW0y3L7uqKdG/Ul7R2eHZFr0YIAI8t6Yz64die9NP7p1W5O9QP1t6ABFIAZxe2nQ7emvgGGQRBkENMkEEQBDnEBBkEQZDDXDXIBJJZnOmZzRJY3nINQzl3z5PJWD6cke8HOOaxpB+y3ggAJTJd4LxIb6Es1v1GNAbjkmOY25+9GFVWvfXKTMkbJ9KVuLejgaMNdqkvfJG8RaOc48yEdTLnnGd95GdOviLnPbL+5pkw8AJV1R3dl/Z99jq3H9ZC5Rb0DdW84hiODPU2vr8A298j6ZSm4YLbpvE04Zq++CPSpy9s28W3yvRw1st2BbTtXkOVt1p6tYL9tjbUBYDG1HGGrmBqiW+QQRAEOcQEGQRBkENMkEEQBDnEBBkEQZDDfBPFBWpKdvOHKTAilOvsJXSXuxRYcFbEY9OCcXm2U3P7DAnkj2hlvl61De1fWVTlyg2OOjl9o23lPfrc2rGfY2MKEI0W7cCwy7LBi2ZxkjclWntmFSXHKEPVca4HO4izOzi7uQPAuDbbEVp6ur8cFDPJ/7DDICO9obLruMB3OLFd/77knDMHMThoU+7Ytwgq+9SOc//LjJcgijxnJfJuyOxCgmbs2C28v+KsMknP2WBHJ3APHOfvq3WdTH6quWf2ud7Rqxbu7umyF0xUq2sWdF2Pb5BBEAQ5xAQZBEGQQ0yQQRAEOczdrGJaBvBWrmPtptSz+8xiVPNEF9qHpI/euhUl6q/WL9L/40e/qso7Q52sCgCf7L5OlcctvQ+bvgL2nFlnYr0OcBLdR/ZSsgGEWZWxUkDTI33R1SCNwQUdwzmuGW0yzvC0tiIf56wFZnT/VKychXKbdD4elqGTwM3jP2PVTAAQSqjn8XduJ9N/d1VDfrGArvPYWYky8SqSrEna3GyTgO4Z5DKV7VsnZA82bEPlAqsa1sv6oWk29UB1e1bPnT6uFFz6Mb5BBkEQ5BATZBAEQQ4xQQZBEOQw9zzIaUPTcsfTs7Q20F/W+wz1O+kAgEROFJ62ybl4w0XdzuCk1UL+zulLqvy6xkVVvtg/Yercf0Lrlt+hvK/ypmNsS/rbiHIAS07SVkZjV7FrHyGVKNfTmG04C3DxRyabfHjSDRuKGFMJp44xzKXfe/o0nbOnzWa0KFpDe68aUwkAKPco73FPC3L1rY6pw2PbPc0Cor3OS9+h3M9M71NpO7mstLAaG/UCQJ/MeQfLLMR6JhiUn+gYcjCce8u5qyXHCERmGAlXFmcvntUe2lzJx1b0hX3kzFVV/vremVse09M13f0K7RUEQfD/ITFBBkEQ5BATZBAEQQ4xQQZBEOQw5yBNwnhqxb5hw1ktbp0CFpSM6iW9ssty1p699CEn1y6sWyH+VQtaCG6PdcDl8zsPmzrX21qsb2y0VbnbtIIzbuhtHLzyEohLJJBzcAKwieJmUcYiARdzTFuFr0mRFSM5AX2WSQkAlFt0nR19nxOcMzIyGdnF7sCRqOZl3TgHZABg1NQRL3b+Xn7euj1UL17Xx81oxcVFe6FbD2nzk1RyHN05HsTX3UsU55cEvBUuGa7DwZ+u7duIjL3HTTrG0Aazdjq6UlayEbvOSI//Zl9HmS61Vkyd3pRVf895scIjvkEGQRDkEBNkEARBDjMnSBF5QEQ+IyJPiciTIvLeyfZ1EfmUiDwz+X/tle9uEATB/Cjyh/gQwC+nlL4kIksAviginwLwLwB8OqX06yLyPgDvA/CrtzxSCcCU6elgydETyfiAzUpZ0wCcRfQ8s1U60+GSFr02GlYz2iJd4zNXvk+VL37zlO0MsfbwDVVebjrtXNtQZV5lz9PNRg1KDl6yOg2fs2sAQRi9in0onKRjo0WRvpjxqodwXhLgZp1E8TINnWcE2yfpqb9KWqejW1Z3dXnnVfomGyzahmrbZKhAmmn7lH20aifv03UGMwRfAL1lPXaeWYUx6KCE7RHrfgASJX2zCQm/vAAAY+4fX8KK89wt6oFprOqLWKt4zsKa662m2XZhS38fG7RIE27ZgSqdPGx7OCz2x/PMvVJKl1NKX5r8vAfgKQBnAfw0gA9PdvswgJ8p1GIQBME9wsvSIEXkYQBvBPA5AKdTSpeBg0kUwOyvU0EQBPcQhSdIEVkE8KcAfimltDtr/6l67xGR8yJyfrTXml0hCILgmFBoghSRCg4mxz9IKf3ZZPNLInJm8vszADa9uimlD6aUHk8pPZ4tOU4TQRAEx5SZQRoREQAfAvBUSuk3p371cQDvAvDrk/8/9nIb5xXnAJv0OuLV+5yAhRGHZxtlA1W9U8VJRt3s6iTda/taLM5azucLbdre0R8KyRGHm1f0tvo1fT7DhuPgskIryq06ji3GJXq26xE7TZtk82y2A5BN1nZcdyjgYoJojls1tz12AkZDCvzxvWFWjARMtIcdpLzgydJFcgOv6+Pun7PnvFvVNzdfn9q20zV+HhpOZwge/7Gz6qQJcPFKiN5Jz8glT86bB6WaPsmza9rt6tEl7cIDAMOx7syNvk2gv1xdVuX9up4Y+kt2aju9emgnv1kuELFEsSj2WwD8cwBfE5GvTLb9GxxMjH8kIu8G8AKAf1KoxSAIgnuEmRNkSukvkf/Z8fY7250gCILjQ7xJEwRBkMN8zSpGgtLuYZP1q1az6Jwip+NVrfWkutUOsh19Gp7T9HCZXKKX9SpoyzWbwM2uwyeXdBT+0kN2+BZodbVypvt/7ZrWNQGrv3VPzF7hj5ObM8cowCQQ0ymWnNX62LGak6THTjLwLDiJGgBIZsKAHN75egHAcDD78zw1qB4lPJccU4+y9hMx2my5a8+5uqNPKuvovpW71pSks0J6ep0aKtmEeh7v/gk7LtLTbfP9Mq45YjPrknxYbzFC0qO5Xa9Oc1E/D48tawOY1y28aOq0x3rsBk52/OaCfonjuX39ssVe3wYrHlg8fGnjqWx2gjoQ3yCDIAhyiQkyCIIgh5gggyAIcpirBilDoLZ1OCfXt6020j5DeXiUr2i0EwCVHT3Pe2YVlVWthfzQ2QuqzOa4AHBjoPMe+2SymZ20/T+7oJPZauQE+9eDh0yd7jKtfEiGv6yRAQD5aFjTVFhDWV7V0EtO4DzTUZ1WrnNyV1nbZO2T9UbAWSGPTDI8HRlszMv3BoBSQ2tL47a+Zp4eWtnXx61S2TMJ7q1VbrkP5yICVk9MC/reGDl5tayHVq85xh/7pBOT/Dly81DJUEQ/Hq7uXRrotssd0o2dFRf3QFrhil4J9PuaV0ydNt2EF7vWKGxMiZxVurCjseMyfQTiG2QQBEEOMUEGQRDkEBNkEARBDjFBBkEQ5DDnVQ2BNKVtt+6z8/NgmVTpGS7GANBfp2TykpPM3NcC88X9VVUeOkp8f6TrXO/qoA0LxQBwua0trTNS2Tn5HACGJ7SinzJ9WXonbZ3m/fu6jnPKrU1yT/LGhcj29TlzAKy6BwMHJEYUJBhbQ2i72l0BEwahAFG2b2/flFEgwbib2+NyUKa5qaNb3qqGzLhMKyNuOgGwqh7LDq+s5zTDt9jKM3afhSv6ArRP6+N2NgqsWMiu/CagZ4M/nnEJw88iryZ4Y2gdvtp0A61WbJRyo6Lv/8tkJf+dnXVT57mdw2TyWNUwCILgNokJMgiCIIeYIIMgCHKYrwYJrUkYY1IA4xWd8Fmq6ozVcsVmsJbpBf7h0CbTDinJ9XpbJ5K2B1Z0GZHRQburE1gXG5RdC2Ch4iybN0WzZn/fXdJtj8hw4bVnbTLtj596QpX3neUe/yB7XJX3rmu9R/bs5a9uk+kCrZLhaVO9NU4Ynp1onSiZn81WvVX1zMqHjmxZ3teNVUgzbWw5xhN7dI/ta01vXLX3kwxGt9zH0+dYlxwu6L72HCOKChn8Vjq2/90Tuu3W/bqd3vpsB+lEt4K3QmFGfVncYSMTe9z6RkeVX7P6kiqfqViX4OuiDV2WPOGYqFGieOYYYLf7hx304gce8Q0yCIIgh5gggyAIcogJMgiCIIe5apBJtBlCycnLEzKYzWhxHfHyCElz9I67vKS1kAodt5JZ/YelJ06HO9G0y9i+buWyKpdJC3lq9z5TZ+u6XoCIxbWHF66bOq+q6kUkXxicMPsk0lnKm1okquw5GjDlurXPJPq9J/zd2kRCKlYPMnmbPdLwnOss4JxG2/8qSVqco9lb9bQnNrTQWvOoZr9HlPq31vW8OkPyT2CTYE93rW+RnrhidsEe+Z8M1smRw+uqWbSLxtu5zNVdfY0WXtLPTHvDnvOI8lBLdOD22FmFj4/hJIjeX7mhyo/WtLa5N7Sa/AutQ9OLb2dFVvaLb5BBEAS5xAQZBEGQQ0yQQRAEOcQEGQRBkMN8E8UrCaPTh4nSzqJ6OLmhM3vrZS04t/o2G7VL2xbqNhn73JJW7zl4Us+sBfRqRQd21uil+Ypju7zGmdXEeMkKzv0zWvweUWa197L+laFW61/sW9flASXHc4Cle9oJnnAQhgJepbZNmq5d1tuETD6Gi/ZCc/J4uTXbRd3kCzvxFhnTKoD0MgIHRg76Qu7Ue3SOXs56pk9gWNflrhMM6q3xNt2X2jX7fYWd4m/8bXvNls7pJS4XKMC1u2fdtcf7+pkptXRDmbP6Iyfdl1u6L7WKrbP7onYqefKEDlKOncFlI4m9oQ3knKpps4oH6jqQWXaezUcXr3735/OetbxDfIMMgiDIISbIIAiCHGKCDIIgyGG+qxpmY9QXDw0eahWrA7Dm2B3O7uJyU4tTNSfpe7OtV1dr9XRG9IJjIvHwstY1lsu6HTb2BIABiUYPVq+p8tuXv27qvHnxW6rM+uJ95R1TZ2+sE2G3eJlDANWqHsveqi5L2epZGW1LvIpky2qQVS2BoX5dH2Ow4OiupMXy6n3+qoDcrrN6ZefWGiQnwgN2pcOFC1pHzq5rvQsAxot6/NsP6WT/3oodp8ZVWj1xV/fN8RtBh8ySKyc7Zp/1BS3Y8gsCfce8pb1LGiSZEXv2xQO6xbb/Fh1j4Bg7v6iP+/zKSVVu9e0F4Zc2ru1aU92MEr0fOaGf1dN16+y8Xj28rpywnkd8gwyCIMghJsggCIIcYoIMgiDIYb55kEkwHE7PybObH7HhgmOEuVLT2uBixRrZsga5c0PrGttDbdIJALsbOv/q6rKuc6ZJ4huAMzWtF1ZEC1z3ZbbOUkULbtuVq6rMuiYAvDjUeY8nndW01ptar+p2yazCMR9mo48e1Rk6ZhX9Zd5S4HOXduFUt2QlVZRIJmZNDwDq10l3Xdb994xsazdoYbVtymVtW91PGrrDQkm9lX1HH23rbZ11PQits6YK5CGtLzadHN/NXX3vjkb6uCNHg2SDkeEy3QuORFeivNrSkA1z7dhyOmL9gtYcr+06Jis1anzBxioee0CbU/zQ2guq/O22Pe4ze6e++3N3HIt2BUEQ3BYxQQZBEOQwc4IUkbqIfF5EvioiT4rIr022PyIinxORZ0TkoyLiJFAEQRDcuxT5BtkD8LaU0usBvAHAO0XkzQB+A8BvpZQeA3ADwLtfuW4GQRDMn5lKZUopAbiZKVuZ/EsA3gbgn022fxjAvwfwO7c6VpaNsb58KDpzUjhgk0QzyiDmJFgAqGf6ONWSDT5sNHSy7+A0rSg3sENxghJwX7/2oiq/rnHR1OGk7vvKOniy7rwkXxd9Ti9SAvr59qOmznMdnXC7PbCGBPWyDv4sNLTAzwEwAGi3yU27RdnZjjt49yQ7feuxbW467uBkKjEuU3Kzs+IlBw5GNbtPZ0NfRw6MNL9h+1/fpKBeSfc/LdlE5VSm1R87+p5buGLbGVd0nXGVyhycAA5s+KdgYxYA6LUpENUmh/Se/R5UYTd2NhR3hr9+TW9cukBu/073h3SNKm1yGB/YvnVPzXakr9HcMRiT2YbTmfLUvFBsTcOCGqSIZCLyFQCbAD4F4FsAtlNKN3t5EYATgwuCILh3KTRBppRGKaU3ADgH4E0AXuPt5tUVkfeIyHkROT/ccTysgiAIjikvK4qdUtoG8FkAbwawKiI3v8ufA3App84HU0qPp5QeL680vV2CIAiOJTM1SBE5CWCQUtoWkQaAH8NBgOYzAH4WwEcAvAvAx2Yda6ncw1vvOzRm2KhYE4AuORK0yV3get/qQdMvoQPASmYTe/k49zW0Nni9byfvpbLWpk5XdJL32Pl82RtrLfBKVxtPXB/ZhPSlkk507yY9Bi8NTCY2rtI4XNp3lrsj6lWtSe51rDvCuE9JxQNSa3j1O1ijg6ULWjNaedomx2NM2nJV34rdU1ZTHZEpbanvJGPvkyHHSO9T3rH3RmmPtg1Jw644j0lH67mlpr6/ZOSYEZPJLhvzytAqY6OhriNVR7evkx5Hvx+L7b/wccmUxPGbNea97ZNkGty0/ec6lb3ZJhFplc7AWYDw2a0NVW6W9fWoOlp/SemSxcwqiqSTnwHwYRHJcPCN849SSp8Qka8D+IiI/AcAXwbwoUItBkEQ3CMUiWL/DYA3Otufw4EeGQRB8D1JvEkTBEGQQ0yQQRAEOczVzackYzSzQzH1oepVs88DFe3AvTXSAYorAxuMWKLl7thBBwAu9rW7x/WBDnKsVKx4X6EkdQ6WcOAHsC7jz7Z0QveVlg24rNZ12+yGvDOwwZSdng5ilJzE2OGYXF2onDnOSIkOU72mVXZ2wQaAjPOsh/q4o4ZNbuZM3VJfRwWa37GBHU7OHqx7gRzd38GCrlMv2/5XKeAifQ4S2HEScs8WGrhx1Tro8DmWyf1cxrZvJXJ4F+c687ZZQRsAGFPCOT8y4hgAcWK+Se5fsX0b1SlIRqs9jmpOBIYcpcb8sgKATkdPXc81rHsP05xyzRqOnRN0iG+QQRAEOcQEGQRBkENMkEEQBDnMVYPcH9bwf68eGi9cWbJ63D898XlVbooWuB6gVQIBYLWkX2HkZG3AunL3RvrUW84KhUyJNMm6s/Rej5yKVypaH4XNc8eYnAG2ujqZfKtjK93Yn/1WUrlMZgIz2gUA9Ehz3CFHd+dt0SF1b/chfYzuqu1rtaV1ptoNLYI1nnMaemFL19ldN7sM7tdO60Ny/mZ/rgBnAAAV1klEQVTDCADGnIJJVauB8bbBoi6zFgoAWV/fP11yFB8uOaYMjus7Q14n5rqzrgwAgxr1b4XacV4I6Jb1WPILAqMFR9Ou6+Nyu8lpp0LJ8Dai4Gux0+y1rW4/mNKFR6nYd8P4BhkEQZBDTJBBEAQ5xAQZBEGQw1w1yNG4hJ3uoTZwobRm9vnvmX6rcYeMYBuZ1f0eaWhtyjOr2Bnq45TJVHcBdrW4AekULVp6z+vLWqV9y3083e+lrl7Cb7+v2+k4JqlZRivxOZoM5z0OaFW6Qccet9Sm3EmSclhvBIBhk3LdOnSOLJIBSHTnVVpkUtuwmrCQ2UNyVhss0RKLnJ84qtrvBKM1rZGWqO1UsXriqK5PoLemy/0l246M9HHap/W4pKbVG9lgpFmz92l/SHp6V/ffXdWQ7sOFZVoZtG5XBr3S1c+rbNNFdORSY3bb0X2pbNu+pR19k1UcEbJ/gnTWNTJtzhzD4qlz9nRZj/gGGQRBkENMkEEQBDnEBBkEQZBDTJBBEAQ5zDVIA+hVCa+2reK/339AlbfJ9ZpXPQSAbzW1u/CJesvsM6SACwc1+iMrFu85JhHTeCYSDzWvqzIHaWqZs6oh7dPL9GVZdgRzZq9ngxrdng7C9PfJ9bplz3lc1+J25wEab+cjNdu5dXJ51nEUcV5Fj7oyXLPJ/pXhaapkj8uJ4KlEBgvOJR0u6HEar1CQo25PelzRxx00aYVCx59jTI7bw0Xd/6xu7w0OyixWbJCmQ/cyB2lcquT63tBBGl41EIAJ7JTIAT3r2WDcuKMDjvy2AptZANbNPNt3bjqqdnZJrybad8wopucbL6jpEd8ggyAIcogJMgiCIIeYIIMgCHKYr2FuKWGxdqineTrAAmksXUqCHTmmomMSNjz9YamidTzWBjsjKxo1y57V6CE1Z+W0s7VtVd4oa+PXbs3qQ72mbrtLWdQv9mxC/SYll79UWjL7DElX7e+RBukZtK7o8V9Y0NpUp+sYN9zQWnK5ra9r45pN2i139LbaNX19smt2xUtcvaGKsmrNTrK27l/92q01SQCQge5Lf0XrZgNvtT7WNmlYWKMEAMphx2BD31/3r1uT4O5A3wvXdq1uP3au4zSlkmMIsajHm587z0yZk74rBYxMeuQn0jutn5lsyT5jiZrulWpmH9Yyu2Q+w2Y0gH7hwjVqcYhvkEEQBDnEBBkEQZBDTJBBEAQ5zFWDzGSMtdqhULFYsfl9q7R4FucI9h1toUq5hWtVK4acrGpNiw10PQ1yu69z8Varum9vWLpg6ry1+U1VZp310tAuOsawAfALVWsM+2T5nCp7xhmcc8m60l7T5hqWyaB1QEYHFcfAtX1Ct91us9ZpP4c5/a20rOtke/Z6yII2lRhtWA2yc1qfU39Zt13uOoub1bUe1Tqj6/TWbR32W+XyyDGeKJ0g8+cTOnfvVFMv1gYAL+xq/dk1nuB2yKihXLVa+caSzhVeqel7uyxWg2wsaT16QPmifSuVY0g6a3NFt7PcJENpACs1ve3bVbsgV5+05ku7tKBeyya8jqaMMsbDMMwNgiC4LWKCDIIgyCEmyCAIghxiggyCIMhhrkGasoyxNhXoWK9aU4kHa9rsYbmsBdvtgV0hjxPFB06i+Es9LeK2hlpg5kRTAOgMtRB8uq5F9MdqV0wd5gudh1V55HwmvbXxrCq/pqrP8fTgRVPnpcGqKu9UbMDlTF0HASrrOnDwTOuUqfPUNb1tb18fd3HBiurlE/o67g4oMNKyY5t19T6Vlg4KSMu6hQ8v6nHIOnafWlmbnciYgj89G3zIeuSiXqX7acEmFfdO6bEsb+i+PEQBGAB4eEnf2xwY3B/YhOgqmbOsrdhnZr2hg3p8/19v2WdmQIGzLvVl6ATWEiVX8yqM5R0nGPeCHv/OSd23+oM2uDggB3cOyABAeVMfd/AdPXYL+p2Cg+NMPTLSj0TxIAiC2yImyCAIghxiggyCIMhhvhpkaYyN2mHC9iO1LbPPw9WrqszGDZe7NtGaE6IbmTUV5YTt5bLWjDpjayLBhrm8+uDzfW3UCwD1mtZU9sih1TOeOFvR2tSjFa11ttlNFsCIdNcKu4wCqJd0X64OFlXZ07x6A633DHb1uOxetHoWk5E5QvuM1f36K2T2UNbXue60IzXdX6k4rrSOGcU0g0VnLGv6ewIPZcnxLJFVfY+99sxLqvzqZV0GgN5Yn+Ollr6Xb3Stjtys6MZX61Z3PVnXL0FcbGl9es9Jmm519HVdrJJZhZMoziss9sjYRK7bdpaf0/dCel6P9f5l+wy16RKd+5q9t5ee1s8MevTMV+zUtvl3D9vyrqlHfIMMgiDIISbIIAiCHApPkCKSiciXReQTk/IjIvI5EXlGRD4qIgUWwgiCILh3eDka5HsBPAXgZkLhbwD4rZTSR0TkPwN4N4DfudUBGqU+fqB5aPCwULJmFa2x1pn2R1rXWK1YIwrOlfT0uCvkVnqVyryoF2BNNTk3rD22Gt6jFW16ejJ7SpX/T+lVpk6X9M8vU17e/2r9gKnzV9cfVWXO6wSABykZbFr/BXyzkNaW1v6Wn9I6X+2GNW6gS4Rxpvs/sJ4S4FTV0pC0qobVF7P779PHWLKa3aimD8zGtcOGvc6Dht5n/yEyYF6xety4p9t5YUfrfstVqxWyocjppr5XvMXmuM7AuU85n3KnSxfEMYctl/U5LVX0M7Tm9L89IHOKpp4+WvdbfXeX6jS2yEx5095PHEKoX7MxBdYcB2f1+HdP2OehdfZwHLxF1TwKfYMUkXMA/hGA352UBcDbAPzJZJcPA/iZYk0GQRDcGxT9E/sDAH4FwM2PnRMAtlNKN8PHFwGc9SqKyHtE5LyInN+7XjB0FARBcAyYOUGKyE8C2EwpfXF6s7Oru9BsSumDKaXHU0qPL60X/F4bBEFwDCiiQb4FwE+JyE8AqONAg/wAgFURKU++RZ4DcOmV62YQBMH8mTlBppTeD+D9ACAiPwrgX6eUfkFE/hjAzwL4CIB3AfjYrGN1RlV8pfXgd8u7Q5tYWqEE1YttLb5yUiwAnKrqxOpX1y+bfX54QRtCPN8/qcpf3n/I1Lm4p9tuJ/0N+Mn9+02dR2ub1I5OhP3zl15n6uz1dbBnjZKBvQTiva6z0htRKelgVWukhesLdH4AIGzcQM10N7wV/nS5tq3/mKharw3zN0hzSyf7l3ZtkCBR8Ge45gRp6rSKYaYbkpH3h47eZ1TV+4wdd3DQOG1f0GP55ZENWLx6Q98br1vW96kbXOzpZPJO3ybQ7/b1c7TX1uVpJ+2bpCq78Ovx/r4Fm+jOPFfSTt/e6opDCrZV6PFtXLdu5zwM5Rv2XpCuE7iZwollYTx9XYt5VdxWHuSvAvhXIvIsDjTJD93GsYIgCI4dL+tVw5TSZwF8dvLzcwDedOe7FARBcDyIN2mCIAhymKtZRXtUwd/cOMwGut6xesqJpk6WvdZeUOVL+zbruLWqtTVPyzld0QambFx7omq1zT7pSNu7ur9f6tuo/N5Qi3ZXWrq/F1+yZhWpTy/wb+gxGI7s51ivp9se9azO9NQ1PXaVBa3bVKt2nNKi3ta5T4s1WdeKNyU2HxWqYyUklAZa5xss6nOsN2yir/S1XtVdt/t01yhJncxuk3PH8/sKK8+QBvkdW6m3xrol/X7Pmqp8cVNfj2+cPK3KvOokADRrZIqxZrXBcl1fs2/26bg37H3aI0eIvxzrFw++tWZNJLb2dP87exRD2LPjtPqCLrPWnPXsPSj00oCrG5MZRaJ7rty2Y1luHZ6z48XhEt8ggyAIcogJMgiCIIeYIIMgCHKYqwY5GGV4cedQm+l1rTbSIV2vtU25bn07p1/d0jrfX+NRs49kWnRYWNIv55cd/Ydz20odrXNsO28GPTHQQ9rZ1TpNdt0ZcpLwdjPSZkdO0lamdZlSebahAhsUuAxJs6N2hgtWD0on9HH7pInByY9j3XXvhh6X7prV8OrXdTv7Z63u2jlJ/aWFpcSm3WHxBd2XhU29U7lldbL2aS067j2oj+GZIWS7ZO5Q1vf2+iltXgFYzfGHV54z+7ChS/tB3bcnKmdMncElrSeOXtDli45hce2aPsdF0m49rXnxMuW39ul6jB19kR7x3ulFs8tgWd8fe+fouXIOO1g63Oh4ULvEN8ggCIIcYoIMgiDIISbIIAiCHGKCDIIgyGGuQZrxIEP78qHgmu057shVLTg3t/Q+ZWsoDpRmn8aQYj3dZd1OqWcDCUvakNsI772hk4C7rftSo3Os0WJsAEALLqJDK871122QIFvR3pqlzDFU4L5RUGzcseOW7Wn1WihANKo56ndZbyuTEYLnaD0U3c6QVjncfdSq6C0KygycgNG4oYMAifrGZhwA0Kd40P79tMLidVtnVOXkeF30glnls/rmffzcRVV+69ozps6bG98y25gLw3VVfvUZbaz11RVrxPLJZW2acqOlH5DBwI5/e/nWBimlPWfFyAY9D9f1PpWOM04duobOSpW7D+rj7D1G979zm6bG4T58X+QR3yCDIAhyiAkyCIIgh5gggyAIcpirBokRkO0fzsnVHastjEgrLJGcVXKkNulR0rSTDFzRnrpGC5Tk6FnZrXWmitP/jFbRIy9fNLdssrZ9cV5/blV37GWSEW1zcsnZQGHUoJUDnY/HrHPr/vMqgQAwqumGxmVddq8HDzcPtZPTzknelT3bF6HlEnkMppOFb8J6YYdMgYcNq62Nafh5lUbP1GNwVd/cXyvrBO6tjk2Ifn5dm0acrd0w++wMdVJ3M9MZ3N9oafMKALhGxhOsOSbn5QThlzToGpU7tk65pcc268/W/lhzHNXscXtadkXttNZ3R0N7cw/7U+c4B8PcIAiC72liggyCIMghJsggCIIcYoIMgiDIYb5BGoESR8dO3ml/RYu4/VWtBJcGjqM1b/MEftKG2XTcC1gMm5SwWkCIL5nF1shduzc7MXZEgZCB1tNdePU+wJ5jiYT3oTVsMefIQY7MWUyu3ObgD60k6Di2jMt6H8/9hjGBG0do5+vI5+O5uPAqhokSxzlweLATr5aof13Zt52r7urGx5d0Q5cq1sHohfpZVfb6XxrwBio618wE4zjm5wRDKfZj8v/5GQPsdeUAmEf9hj5QpWUf6MUX9D6ttKTKZR4TAOP1qeMMi0Vp4htkEARBDjFBBkEQ5BATZBAEQQ5z1yDHUxnCrG8VwdMKx5x17DhYs34lvIsjSRhNhXUZL+eV+jekhd+66/YEsq7e1lvVnelrY/ODpgs4InNiNWtVZbuQo9EYefXBo+All5v+0y7edR6RZj327p8Z0pJZgREw15Hb5nYP2iETDE6Gd8xPePzNSxDa5B4AUN2erWFX6DqWu7Od4/maDBq3fikCcPrLqw86zXId1qOHThI4G1g0rtiBWXpaC6ILL+lVBTob9gHZnlrh0tNLPeIbZBAEQQ4xQQZBEOQQE2QQBEEOc9YgE8b1Q6Eilez8nGq06hnl7jmeEmblPZQKCAxFNAjOneQV/4zjgtUc2dTAM/8s02pwgyUqLzvijsnrdHRX039dLjm5YEM+JSPWHoEjyJhjx9A0UU6d48NrdDDWHL2cQCHNmk2B3RxNHlxjsODkfrJmyrt448TSoHOdM5LoSmx268muMzRsN3eS25mhcQNAybQzO8e3v0Q5smWbiMp5kBmtljgu2xMcTj1HsaphEATBbRITZBAEQQ4xQQZBEOQQE2QQBEEO8zerqE4Jpd7KYiR+Jw4SeD3mII0Hi9tF3rSnfVKVox62jtR1Njk3O1x31GEOlvC4eOfHdZzkeEORoAB/ZJpIj1OHt83OU5790ewdo0Byv6lHF0AGBb4T0PibIKDH7efT+xS4rBy4ETZv8W7tWdfZjYDRuPB9epSvW1XHYT/T2244q29Wr+roDpuDcHAU4P7GqoZBEAS3RUyQQRAEOcQEGQRBkIMkL/P6lWpMZAvAdwBsALg6t4Zvj3upr8C91d97qa/AvdXfe6mvwPz7+1BK6eSsneY6QX63UZHzKaXH597wEbiX+grcW/29l/oK3Fv9vZf6Chzf/saf2EEQBDnEBBkEQZDD3ZogP3iX2j0K91JfgXurv/dSX4F7q7/3Ul+BY9rfu6JBBkEQ3AvEn9hBEAQ5zHWCFJF3isjTIvKsiLxvnm0XQUR+T0Q2ReSJqW3rIvIpEXlm8v/a3ezjTUTkARH5jIg8JSJPish7J9uPa3/rIvJ5EfnqpL+/Ntn+iIh8btLfj4rIEVYqemUQkUxEviwin5iUj3NfnxeRr4nIV0Tk/GTbcb0XVkXkT0TkG5P790eOa1/nNkGKSAbgPwH4cQCvBfDzIvLaebVfkN8H8E7a9j4An04pPQbg05PycWAI4JdTSq8B8GYAvzgZz+Pa3x6At6WUXg/gDQDeKSJvBvAbAH5r0t8bAN59F/vIvBfAU1Pl49xXAPgHKaU3TKXLHNd74bcB/HlK6fsBvB4HY3w8+5pSmss/AD8C4C+myu8H8P55tf8y+vkwgCemyk8DODP5+QyAp+92H3P6/TEA77gX+gugCeBLAH4YB8nBZe8euct9PIeDB/VtAD6BA9uIY9nXSX+eB7BB247dvQBgGcC3MYl/HOe+ppTm+if2WQAXpsoXJ9uOO6dTSpcBYPL/qbvcH4OIPAzgjQA+h2Pc38mfrF8BsAngUwC+BWA7pXTTvP843RMfAPArOPQHOoHj21fgwJ7mf4jIF0XkPZNtx/FeeBTAFoD/MpEvfldEFnA8+zrXCdIzbooQ+m0iIosA/hTAL6WUdu92f25FSmmUUnoDDr6dvQnAa7zd5tsri4j8JIDNlNIXpzc7u971vk7xlpTSD+JAwvpFEfn7d7tDOZQB/CCA30kpvRFAC8flz2mHeU6QFwE8MFU+B+DSHNs/Ki+JyBkAmPy/eZf7811EpIKDyfEPUkp/Ntl8bPt7k5TSNoDP4kA7XRWRm+59x+WeeAuAnxKR5wF8BAd/Zn8Ax7OvAICU0qXJ/5sA/isOPoCO471wEcDFlNLnJuU/wcGEeRz7OtcJ8gsAHptEAqsAfg7Ax+fY/lH5OIB3TX5+Fw60vruOiAiADwF4KqX0m1O/Oq79PSkiq5OfGwB+DAfi/GcA/Oxkt2PR35TS+1NK51JKD+PgPv2fKaVfwDHsKwCIyIKILN38GcA/BPAEjuG9kFK6AuCCiLx6suntAL6OY9hXAPML0kzE158A8E0caE//9m4LsE7//hDAZQADHHzSvRsH2tOnATwz+X/9bvdz0te/h4M/8f4GwFcm/37iGPf3BwB8edLfJwD8u8n2RwF8HsCzAP4YQO1u95X6/aMAPnGc+zrp11cn/568+Wwd43vhDQDOT+6F/wZg7bj2Nd6kCYIgyCHepAmCIMghJsggCIIcYoIMgiDIISbIIAiCHGKCDIIgyCEmyCAIghxiggyCIMghJsggCIIc/h8NhP305HzD4QAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "# This is not used any further, it's just for visualizing the features\n", "evaluator = K.function([content_model.input],[content_model.output])\n", @@ -168,19 +330,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "# Please call this second network 'style_model'\n", "\n", "#! Change me\n", - "style_layers = []\n", + "style_layers = ['block1_relu1','block2_relu1','block3_relu1','block4_relu1','block5_relu1']\n", "\n", "style_base_model = vgg.VGG19(input_tensor=kl.Input(tensor=K.tf.Variable(style_img)))\n", "\n", - "#! Change me\n", - "style_model = None" + "style_model = km.Model(inputs=style_base_model.inputs,outputs=[style_base_model.get_layer(n).output for n in style_layers])\n", + "\n" ] }, { @@ -192,7 +354,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -209,16 +371,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# Please call this third network 'blend_model'\n", + "blend_layers = content_layers + style_layers\n", "\n", - "base_model = vgg.VGG19(input_tensor=blended_model_input)\n", + "blend_base_model = vgg.VGG19(input_tensor=blended_model_input)\n", "\n", "#! Change me\n", - "blend_model = None\n", + "blend_model = km.Model(inputs=blend_base_model.inputs,outputs=[blend_base_model.get_layer(n).output for n in blend_layers])\n", "\n", "# Separate the model outputs into those intended for comparison with the content layer and the style layer\n", "blend_content_outputs = [blend_model.outputs[0]]\n", @@ -242,14 +405,32 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(1, 400, 543, 3)\n" + ] + }, + { + "data": { + "text/plain": [ + "[195710720.0]" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "def content_layer_loss(Fp, Fx):\n", - " #! Change me\n", - " _,h,w,d = Fp.get_shape()\n", - " loss = 1\n", + " _,h,w,d = Fp.get_shape().as_list()\n", + " X = 1/(2*((w*h)**.5)*(d**.5))\n", + " loss = X*K.sum((Fp-Fx)**2)\n", "\n", " return loss\n", "\n", @@ -259,6 +440,7 @@ "np.random.seed(0)\n", "input_img = np.random.randn(1,img_rows,img_cols,3)\n", "content_loss_evaluator = K.function([blend_model.input],[content_loss])\n", + "print(input_img.shape)\n", "content_loss_evaluator([input_img])" ] }, @@ -281,15 +463,28 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "113934860.0" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "def gram_matrix(f, M, N):\n", " # Accepts a (height,width,depth)-sized feature map,\n", " # reshapes to (M,N), then computes the inner product\n", " # !Change me\n", - " return 1\n", + " f1 = K.transpose(K.reshape(f,(M,N)))\n", + " f2 = K.reshape(f,(M,N))\n", + " return f1@f2\n", "\n", "# For a correctly implemented gram_matrix, the following code will produce 113934860.0\n", "fmap = content_model.output\n", @@ -315,16 +510,30 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[220990.36]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "def style_layer_loss(Fa, Fx):\n", - " #! Change me\n", - " _, h, w, d = Fa.get_shape()\n", - " loss = 0\n", + " _, h, w, d = Fa.get_shape().as_list()\n", + " __,h2,w2,d2 = Fx.get_shape().as_list()\n", + " Fa = gram_matrix(Fa,h*w,d)\n", + " Fx = gram_matrix(Fx,h2*w2,d2)\n", + " X = 1/(4*(((w*h)*(w2*h2))*(d*d2)))\n", + " loss = X*K.sum((Fa-Fx)**2)\n", " return loss\n", - "\n", + " \n", "style_loss_0 = style_layer_loss(style_model.output[0],blend_style_outputs[0])\n", "\n", "# The correct output of this function is 220990.31\n", @@ -343,9 +552,20 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[177059660.0]" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "style_loss = 0\n", "for i in range(5):\n", @@ -368,7 +588,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -385,9 +605,20 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[array([3.5509787e+11], dtype=float32)]" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "alpha = 5.0\n", "beta = 2e3\n", @@ -395,7 +626,7 @@ "\n", "total_loss = alpha*content_loss + beta*style_loss + gamma*tv_loss\n", " \n", - "# The correct output of this function is 1.7715756e+12\n", + "# The correct output of this function is 3.5509787e+11\n", "np.random.seed(0)\n", "input_img = np.random.randn(1,img_rows,img_cols,3)\n", "total_loss_evaluator = K.function([blend_model.input],[total_loss])\n", @@ -412,11 +643,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /home/dunbar/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use tf.cast instead.\n", + "(?, 400, 543, 3)\n" + ] + } + ], "source": [ - "grads = K.gradients(total_loss,blend_model.input)[0]" + "grads = K.gradients(total_loss,blend_model.input)[0]\n", + "print(grads.shape)" ] }, { @@ -428,15 +671,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[3.5509787e+11] -728.9879\n" + ] + } + ], "source": [ "loss_and_grad_evaluator = K.function([blend_model.input],[total_loss,grads])\n", "\n", "np.random.seed(0)\n", "input_img = np.random.randn(1,img_rows,img_cols,3)\n", "l0,g0 = loss_and_grad_evaluator([input_img])\n", + "\n", + "print(l0,g0[0][0][0][0])\n", "# Correct value of l0 is 3.5509e11\n", "# Correct value of first element in g0 is -7.28989e2" ] @@ -455,7 +708,26 @@ "outputs": [], "source": [ "import scipy.optimize as sio\n", - "sio.fmin_l_bfgs_b?" + "class evaluator():\n", + " def __init__(self,in_image):\n", + " _,self.h,self.w,self.d = in_image.shape \n", + " def loss(self,x):\n", + " x = x.reshape((1,self.h,self.w,self.d))\n", + " loss,self.grad = loss_and_grad_evaluator([x]) \n", + " return loss.astype('float64')\n", + " def gradient(self,_):\n", + " return self.grad.astype('float64').flatten()\n", + "Evaluator = evaluator(input_img) \n", + "outputImg,_,_ = sio.fmin_l_bfgs_b(Evaluator.loss, input_img.flatten().astype('float64'), fprime=Evaluator.gradient, maxiter=500,iprint=10)\n", + "outputImg = outputImg.reshape(input_image.shape).squeeze()\n", + "\n", + "outputImg[:,:,0] += 103.939\n", + "outputImg[:,:,1] += 116.779\n", + "outputImg[:,:,2] += 123.68\n", + "outputImg /= 255\n", + "plt.imshow(outputImg)\n", + "plt.show()\n", + "\n" ] }, { @@ -486,7 +758,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/neural_style_transfer.py b/neural_style_transfer.py new file mode 100644 index 0000000..306e439 --- /dev/null +++ b/neural_style_transfer.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python +# coding: utf-8 + +# # Project 5: Neural Style Transfer +# ## Due May 2nd +# +# ## What do CNNs see? +# We have used neural networks primarily for the purposes of object recognition and image segmentation, as these are perhaps their most common uses. However, the conceptual simplicity of the problem statement belie the complexity of the models under the hood. In particular, we can ask: why are (convolutional) neural networks so good at these tasks that we have set out for them? What are they doing that is so different from previous techniques? +# +# In short, neural networks work for abstract cases that would lead to failure for simpler methods. Take "Where's Waldo" as an example. As we saw, template matching was extremely sensitive to the quality of the template, and when Waldo appeared in a slightly different orientation, the algorithm failed abjectly. CNNs on the other hand can recognize objects in a variety of poses, scales, etc. without difficulty. They do this by creating a transformation of the input data (the image) that is relatively invariant to these changes, and that operates hierarchically. For example, the first kernel(s) of a network tends to search for simple features: lines at various orientations, blobs, particular colors, and so on. These features are in turn convolved with another set of kernels that activate on a more complex input with a larger receptive field: lines and colors become simple shapes. At the next layer, simple shapes become something more complex: textures or anatomical features such as eyes. Moving further up the network, eyes become pairs of eyes, pairs of eyes become faces, and faces pair with other features to become, say, a dog. Each of these representations has some flexibility, and this flexibility when coupled with the progressively generated representation of an object of interest is what yields the capacity for convolutional neural networks to tackle objects with the capacity for abstraction similar to the human visual system. +# +# Because this hierarchy of features is generated programatically, *we have access to every level as the network builds up its representation of an object*. We've been interested in the final product as the input to softmax classifier, but here we will explore the internal representation of objects in neural networks through a remarkable application known as *neural style transfer*. +# +# ## Style transfer +# Style transfer is the process of creating a new image that has the style of one image (which we will call the *style image*) and the content of another (the *content image*). Here's an example of style transfer (by a person, not a network) +# +# We have a content image (the KFC logo) and a style image (the Obama Hope poster by Shephard Fairey). Here, the literal content of the KFC logo is reproduced, but with the semantic content of the poster. +# +# ## Neural style transfer +# Neural style transfer is the procedure of performing style transfer automatically with the help of neural networks. While a few mechanisms have been proposed to solve this problem, we will utilize the [original approach](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf). For our example, let's use the following content and style images: + +# In[1]: + + +import keras.preprocessing as kp +import matplotlib.pyplot as plt +import numpy as np +import keras.backend as K + +content_path = './main_hall.jpg' +style_path = './starry_night.jpg' + +# Load image to get geometry +temp_img = kp.image.load_img(content_path) +width,height = temp_img.size + +# fix the number of rows, while adapting the aspect ratio +img_rows = 400 +img_cols = int(width * img_rows / height) + +# Load content image +content_img = kp.image.load_img(content_path, target_size=(img_rows, img_cols)) +content_img = kp.image.img_to_array(content_img) + + +# Load style image +style_img = kp.image.load_img(style_path, target_size=(img_rows, img_cols)) +style_img = kp.image.img_to_array(style_img) + + + + + +# At the most basic level, we seek to find the solution to the following problem +# $$ +# \hat{x} = \underset{\vec{x}}{\operatorname{arg\,min}} \; \left[\alpha \mathcal{L}_{content}(\vec{p},\vec{x}) + \beta \mathcal{L}_{style}(\vec{a},\vec{x}) + \gamma \mathcal{L}_{prior}(\vec{x})\right], +# $$ +# where $\vec{x}$ are the pixel values of the combined image, $\vec{p}$ are the pixel values of the content image, and $\vec{a}$ are the pixel values of the style image. $\mathcal{L}_{content}(\vec{p},\vec{x})$ is the so-called *content loss*, which measures the combined image's fidelity to the content image. $\mathcal{L}_{style}(\vec{a},\vec{x})$ is the *style loss*, which measures the combined image's stylistic similarity to the style image. $\mathcal{L}_{prior}(\vec{x})$ is a prior which can be used to enforce constraints such as smoothness of the combined image. $\alpha$, $\beta$, and $\gamma$ are constants that we can use to tune the relative strength of each loss function. Thus as $\alpha/\beta\rightarrow 0$, style dominates, and vice versa. As with most minimization problems, we will solve this by taking the derivative of this combined loss function with respect to $\vec{x}$, and using some type of gradient descent to find a minimum. +# +# ## VGG19 +# Where do neural networks come in? We use neural networks to define the losses. While it would certainly be possible to ask the algorithm to match the content image exactly, this isn't exactly what we want: rather, we want the features to match roughly, in an abstract way. For example, if a door shows up in the content image, we want a door to show up in the style image, but it doesn't have to be exactly straight up and down, nor does it have to have the same color. It just needs to be door-like in a general sense. To do this, instead of asking the algorithm to minimize the difference between the combined image and the door, we can minimize the difference of feature maps computed at some layer of a convolutional neural network between the combined and content images. +# +# Of course we must select a neural network to use in order to make these feature representations. In almost all instances of neural style transfer, the VGG19 network is used. VGG19 is a fairly old network and far from the state of the art for tasks such as object recognition. However, it almost unanimously provides the best results for neural style transfer, most likely because of its simplicity: there aren't any skip connections or other strange architectural styles, and the resulting features are thus closer to the ideal description of a neural network given above. Packaged along with this notebook is a Keras implementation of VGG19, that also loads a set of weights pretrained on the [ImageNet](http://www.image-net.org/) dataset. You can download the requisite weights [here](http://www.vlfeat.org/matconvnet/pretrained/#downloading-the-pre-trained-models). (Note actually that the VGG19 provided by me is *not* the original VGG19, nor is it equivalent to the pre-trained model that is packaged with Keras: max-pooling has been replaced with average pooling, the activations have been broken out into explicit layers, and the weights are oriented such that it operates on mean-subtracted RGB images, rather than BGR. These modifications provide better neural style transfer results.) +# +# First, we'll need to normalize the images in the way that VGG19 expects, which means subtracting the mean pixel value of the dataset used to train it: + +# In[2]: + + +content_img[:, :, 0] -= 103.939 +content_img[:, :, 1] -= 116.779 +content_img[:, :, 2] -= 123.68 +content_img = np.expand_dims(content_img, axis=0) + +style_img[:, :, 0] -= 103.939 +style_img[:, :, 1] -= 116.779 +style_img[:, :, 2] -= 123.68 +style_img = np.expand_dims(style_img, axis=0) + + +# Next, let's instantiate a VGG19 model for the content image: + +# In[3]: + + +import vgg +import keras.backend as K +import keras.layers as kl +import keras.models as km + +# Note that we'll be working quite a bit with the TensorFlow objects that underlie Keras +content_model_input = kl.Input(tensor=K.tf.Variable(content_img)) + +content_base_model = vgg.VGG19(input_tensor=content_model_input) +evaluator = K.function([content_base_model.input],[content_base_model.output]) +feature_maps = evaluator([content_img]) + + +# In[4]: + + + +# The function defines above provides the output of the last activation in VGG19. However, this is not the layer that we need. Indeed, in the original neural style transfer paper, the authors found that good aesthetic properties were found by matching on the (unactivated) feature maps in the second convolution of the fourth block, called 'block4_conv2' (have a look at the VGG file if you're confused by what this means). We can generate a new Keras model that does this for us easily: + +# In[5]: + + + +# Define the layer outputs that we are interested in +content_layers = ['block4_conv2'] + +# Get the tensor outputs of those layers +content_outputs = [content_base_model.get_layer(n).output for n in content_layers] + +# Instantiate a new model with those outputs as outputs +content_model = km.Model(inputs=content_base_model.inputs,outputs=[content_base_model.get_layer(n).output for n in content_layers]) + + +# In[6]: + + +# This is not used any further, it's just for visualizing the features +evaluator = K.function([content_model.input],[content_model.output]) +feature_maps = evaluator([content_img]) + + + +# **We'll need a similar model for the style image. Generate a second neural network, this time asking the model to produce as outputs the layers block1_relu1, block2_relu1, block3_relu1, block4_relu1, and block5_relu1.** + +# In[7]: + + +# Please call this second network 'style_model' + +#! Change me +style_layers = ['block1_relu1','block2_relu1','block3_relu1','block4_relu1','block5_relu1'] + +style_base_model = vgg.VGG19(input_tensor=kl.Input(tensor=K.tf.Variable(style_img))) + +style_model = km.Model(inputs=style_base_model.inputs,outputs=[style_base_model.get_layer(n).output for n in style_layers]) + + +# Finally, we'll need a 3rd neural network to evaluate the features of the blended image. We'll be comparing feature maps in the blended image to the equivalent feature maps in both the content and style image, so this model will need to yield the output of block4_conv2 and the five layers stated above. Because the input to this final model is going to be dynamic (we'll be updating as we minimize the loss), we'll provide it with a more general input type with the same shape as the content image, but with no values specified. + +# In[8]: + + +# What are we saying about the size of the input image here? +blended_model_input = kl.Input(shape=content_img.shape[1:]) + + +# **Instantiate the third neural network for the blended image as you did the content and style images, but with a concatenated list of layers as outputs and the input as specified in the cell above** + +# In[10]: + + +# Please call this third network 'blend_model' +blend_layers = content_layers + style_layers + +blend_base_model = vgg.VGG19(input_tensor=blended_model_input) + +#! Change me +blend_model = km.Model(inputs=blend_base_model.inputs,outputs=[blend_base_model.get_layer(n).output for n in blend_layers]) + +# Separate the model outputs into those intended for comparison with the content layer and the style layer +blend_content_outputs = [blend_model.outputs[0]] +blend_style_outputs = blend_model.outputs[1:] + + +# ## Defining the content loss +# +# Now we need to actually mathematically define the content loss. This turns out to be rather simple: the content loss is just the (scaled) sum square error between the block4_conv2 features maps in the combined and content images: +# $$ +# \mathcal{L}_{content}(\vec{p},\vec{x}) = \frac{1}{2 M^{\frac{1}{2}} N^{\frac{1}{2}}} \sum \left[F(\vec{x})_\ell - F(\vec{p})_\ell\right]^2, +# $$ +# where $F(\vec{x})_\ell$ is the CNN's output at layer $\ell$. The summation runs over all three tensor dimensions, $M$ is the image width times height, and $N$ is the image depth. +# +# **Implement a function that computes the content loss** + +# In[27]: + + +def content_layer_loss(Fp, Fx): + _,h,w,d = Fp.get_shape().as_list() + X = 1/(2*((w*h)**.5)*(d**.5)) + loss = X*K.sum((Fp-Fx)**2) + + return loss + +content_loss = content_layer_loss(content_model.output,blend_content_outputs[0]) + +# The correct output of this function is 195710720.0 +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +content_loss_evaluator = K.function([blend_model.input],[content_loss]) + +content_loss_evaluator([input_img]) + + +# ## Defining the style loss +# +# Now we need to define some metric for comparing the *style* between two images. After watching the Ng videos, you should be aware that one way of doing this is by comparing Gram matrices. +# +# The Gram matrix is sort of like a sample covariance, except that instead of measuring the similarity of data points with respect to the data features (as does a covariance matrix), the Gram matrix measures the similarity of features with respect to the data points. Stated concretely and in this context, it measure the degree to which two feature maps tend to activate together. It is computed similarly to the covariance matrix, but with the dimensions reversed (and without the mean subtracted): +# $$ +# G(\vec{f}) = f^T f, +# $$ +# where $f$ is a set of $N$ feature maps reshaped to $\mathbb{R}^{M \times N}$ (i.e. flattened along the spatial dimension), where $M$ is once again width times height and $N$ is the number of feature maps. +# +# **Implement a function that computes the Gram matrix** + +# In[13]: + + +def gram_matrix(f, M, N): + # Accepts a (height,width,depth)-sized feature map, + # reshapes to (M,N), then computes the inner product + # !Change me + f1 = K.transpose(K.reshape(f,(M,N))) + f2 = K.reshape(f,(M,N)) + return f1@f2 + +# For a correctly implemented gram_matrix, the following code will produce 113934860.0 +fmap = content_model.output +_,h,w,d = fmap.get_shape() +M = h*w +N = d +gram_matrix_evaluator = K.function([content_model.input],[gram_matrix(fmap,M,N)]) +gram_matrix_evaluator([content_img])[0].mean() + + +# Now that we have a method of computing gram matrices, we can simply compare them between images to assess whether the style metric is similar. Note that we will do this over multiple style model layers to ensure that we are capturing stylistic similarites at a variety of spatial scales. The style loss for a particular layer is then given as +# $$ +# \mathcal{L}_{style,\ell}(\vec{a},\vec{x}) = \frac{1}{4 M^2 N^2} \sum_{i,j}^N \left[ G(F(\vec{a})_\ell)_{ij} - G(F(\vec{x})_\ell)_{ij} \right]^2. +# $$ +# In plain language, what the above says is to compute the Gram matrix at layer $\ell$ for both the style and blended images, and to compute the elementwise sum square error between them. +# +# **Implement a function that computes the style loss for one layer** + +# In[30]: + + +def style_layer_loss(Fa, Fx): + _, h, w, d = Fa.get_shape().as_list() + __,h2,w2,d2 = Fx.get_shape().as_list() + Fa = gram_matrix(Fa,h*w,d) + Fx = gram_matrix(Fx,h2*w2,d2) + X = 1/(4*(((w*h)*(w2*h2))*(d*d2))) + loss = X*K.sum((Fa-Fx)**2) + return loss + +style_loss_0 = style_layer_loss(style_model.output[0],blend_style_outputs[0]) + +# The correct output of this function is 220990.31 +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +style_loss_evaluator = K.function([blend_model.input],[style_loss_0]) +style_loss_evaluator([input_img]) + + +# Finally, we'll need to add up style losses over all five layers that we wish to compute it over. We may wish to adjust weights between different layers for different effects in the future, but for now let us just assume that each layer-wise style loss is equal to $\frac{1}{5}$ + +# In[31]: + + +style_loss = 0 +for i in range(5): + style_loss += 0.2*style_layer_loss(style_model.output[i],blend_style_outputs[i]) + +# The correct output of this function is 177059700.0 +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +style_loss_evaluator = K.function([blend_model.input],[style_loss]) +style_loss_evaluator([input_img]) + + +# ## Prior loss (regularization) +# While this doesn't have much to do with neural networks, it's part of the problem specification. In particular, we want to make sure that our blended image isn't too noisy. To do this, we'll simply add a penalty on [total variation](https://en.wikipedia.org/wiki/Total_variation), which is simply the summed up absolute values of the derivatives. + +# In[33]: + + +tv_loss = K.tf.image.total_variation(blend_model.input) + + +# ## Total loss +# Finally, we can add these three losses up, scaled by some arbitrary weighting factors, to get a total loss: + +# In[34]: + + +alpha = 5.0 +beta = 2e3 +gamma = 1e-3 + +total_loss = alpha*content_loss + beta*style_loss + gamma*tv_loss + +# The correct output of this function is 3.5509787e+11 +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +total_loss_evaluator = K.function([blend_model.input],[total_loss]) +total_loss_evaluator([input_img]) + + +# ## Optimization +# The blended image is the one that minimizes the loss defined above. How do we find the minimum? By following the gradient of the loss function "downhill". It is important to remember that the dimensions of this problem are the RGB pixel values of the the input image. How can we compute this gradient? Tensorflow makes it quite easy through [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation). Note that this is essentially the same thing as backpropagation, but here we're taking the derivative of a loss defined on the intermediate layers of a neural network with respect to the inputs, versus in the normal NN learning case where we're finding the derivative of some cost function defined on network outputs with respect to the weights. The command for computing a gradient in tensorflow is: + +# In[35]: + + +grads = K.gradients(total_loss,blend_model.input)[0] + + + +# This produces a tensor, but what we need is a function that takes our input image as an input, and outputs the loss and the gradient of the loss. We can use the tensorflow 'function' to do this + +# In[39]: + + +loss_and_grad_evaluator = K.function([blend_model.input],[total_loss,grads]) + +np.random.seed(0) +input_img = np.random.randn(1,img_rows,img_cols,3) +l0,g0 = loss_and_grad_evaluator([input_img]) + +(l0,g0[0][0][0][0]) +# Correct value of l0 is 3.5509e11 +# Correct value of first element in g0 is -7.28989e2 + + +# We can now evaluate the loss and gradient for an arbitrary image input. All that's left is to solve the minimization problem. The authors of the original neural style transfer paper use l_bfgs. There is a convenient implementation of this algorithm in scipy + +# In[ ]: + + +import scipy.optimize as sio +class evaluator(): + def __init__(self,in_image): + _,self.h,self.w,self.d = in_image.shape + def loss(self,x): + x = x.reshape((1,self.h,self.w,self.d)) + loss,self.grad = loss_and_grad_evaluator([x]) + return loss.astype('float64') + def gradient(self,_): + return self.grad.astype('float64').flatten() +Evaluator = evaluator(input_img) +outputImg,_,_ = sio.fmin_l_bfgs_b(Evaluator.loss, input_img.flatten().astype('float64'), fprime=Evaluator.gradient, maxiter=500,iprint=50) +outputImg = outputImg.reshape(input_img.shape).squeeze() + +outputImg[:,:,0] += 103.939 +outputImg[:,:,1] += 116.779 +outputImg[:,:,2] += 123.68 +outputImg /= 255 + +#outputImg = outputImg.mean(axis= -1) +#contentImg = content_img.mean(axis = -1) +#styleImg = style_img.mean(axis =-1) + +plt.imshow(outputImg) +plt.show() + + +# **Use this method to find the image that minimizes the total loss. Around 500 iterations is usually sufficient to produce fully converged results.** +# +# A few gotchas: 1. the scipy algorithm expects a vector, while our method expects an array of size (1,height,width,3). You'll need to do some reshaping of input and output arrays and puts the arguments to both the tensorflow function and to the scipy optimizer in the formats that they expect. 2. We have to compute the loss to compute the gradient, but the gradient of the loss doesn't change unless the loss is recomputed. Thus, the functions that go into fmin_l_bfgs_b should be aware of that fact, and should avoid unnecessary computation. Both (1) and (2) above can be addressed by generating a python class that have both a loss and gradient method that perform the necessary array reshapings internally, and that don't recompute the gradient once it's already been computed (stated concretely, the loss method should call "loss_and_grad_evaluator", and should return the loss while saving the gradient. The gradient method should simply return this precomputed gradient.) +# +# Finally, when you output the result of your optimization, you'll want to re-add the band means we extracted earlier (R=103.939, G=116.779, B=123.68), cast the array back to integer, and clip any values greater than 255 before saving.