Max/autoencoder

autopylot/datasets/datagenerator.py

@@ -112,7 +112,10 @@ def __data_generation(self):
         for k in X.keys():
             X[k] = np.array(X[k])
         for k in Y.keys():
-            Y[k] = np.array(Y[k])
+            if "image" in k:
+                Y[k] = np.array(Y[k]) / 255.0
+            else:
+                Y[k] = np.array(Y[k])
 
         return X, Y
 

autopylot/datasets/transform.py

@@ -173,44 +173,46 @@ def resize(image_data):
         )
 
     def obstacles(image_data):
-        if "image" in image_data.keys():
-            img = image_data["image"]
-
-            obstacle_path = np.random.choice(obstacles_path)
-            obstacle_img = cv2.imread(obstacle_path, cv2.IMREAD_UNCHANGED)
-
-            # need to work on the size policy
-            max_size = 32
-            upper_0 = img.shape[0] - max_size
-            upper_1 = img.shape[1] - max_size
-
-            # define random placement
-            cty = np.random.randint(min(max_size, upper_0), max(max_size, upper_0))
-            ctx = np.random.randint(min(max_size, upper_1), max(max_size, upper_1))
-            size_mult = cty / img.shape[0]
-
-            sizey = int(max_size * size_mult)
-            sizex = int(max_size * size_mult)
-            topy = cty - sizey
-            topx = ctx - sizex
-            boty = cty + sizey
-            botx = ctx + sizex
-
-            resized = cv2.resize(obstacle_img, (sizex * 2, sizey * 2))
-            color, alpha = resized[:, :, :3], resized[:, :, -1:] / 255
-
-            # apply obstacle on the image
-            img[topy:boty, topx:botx, :] = (
-                img[topy:boty, topx:botx, :] * (1 - alpha) + color * alpha
-            )
+        if "image" not in image_data.keys():
+            return 
+
+        img = image_data["image"]
+
+        obstacle_path = np.random.choice(obstacles_path)
+        obstacle_img = cv2.imread(obstacle_path, cv2.IMREAD_UNCHANGED)
+
+        # need to work on the size policy
+        max_size = 64
+        upper_0 = img.shape[0] - max_size
+        upper_1 = img.shape[1] - max_size
+
+        # define random placement
+        cty = np.random.randint(min(max_size, upper_0), max(max_size, upper_0))
+        ctx = np.random.randint(min(max_size, upper_1), max(max_size, upper_1))
+        size_mult = cty / img.shape[0]
+
+        sizey = int(max_size * size_mult)
+        sizex = int(max_size * size_mult)
+        topy = cty - sizey
+        topx = ctx - sizex
+        boty = cty + sizey
+        botx = ctx + sizex
+
+        resized = cv2.resize(obstacle_img, (sizex * 2, sizey * 2))
+        color, alpha = resized[:, :, :3], resized[:, :, -1:] / 255
+
+        # apply obstacle on the image
+        img[topy:boty, topx:botx, :] = (
+            img[topy:boty, topx:botx, :] * (1 - alpha) + color * alpha
+        )
 
-            # car detection data
-            image_data["obstacles"] = 1  # defaults to 0
-            image_data["obstacles-size"] = size_mult  # defaults to 0
-            image_data["obstacles-coord"] = [
-                ((cty / img.shape[0]) - 0.5) * 2,
-                ((ctx / img.shape[1]) - 0.5) * 2,
-            ]  # defaults to [0, 0]
+        # car detection data
+        image_data["obstacles"] = 1  # defaults to 0
+        image_data["obstacles-size"] = size_mult  # defaults to 0
+        image_data["obstacles-coord"] = [
+            ((cty / img.shape[0]) - 0.5) * 2,
+            ((ctx / img.shape[1]) - 0.5) * 2,
+        ]  # defaults to [0, 0]
 
 
 # default values for each functions

autopylot/models/architectures.py

@@ -18,6 +18,8 @@
     Lambda,
     SeparableConv2D,
     GlobalMaxPooling2D,
+    UpSampling2D,
+    ZeroPadding2D,
 )
 from tensorflow.keras.optimizers import Adam
 
@@ -272,9 +274,9 @@ def separable_model():
 
         x = SeparableConv2D(192, 3, strides=2, use_bias=False)(x)
         x = Activation("relu")(x)
-        x = BatchNormalization()(x)
+        w = BatchNormalization()(x)
 
-        x = SeparableConv2D(256, 3, strides=1, use_bias=False)(x)
+        x = SeparableConv2D(256, 3, strides=1, use_bias=False)(w)
         x = Activation("relu")(x)
         x = BatchNormalization()(x)
 
@@ -291,12 +293,12 @@ def separable_model():
         x = Activation("relu")(x)
         x = BatchNormalization()(x)
 
-        # c1 = Dense(1, use_bias=False, activation="sigmoid", name="obstacles")(x)
-        # c2 = Dense(1, use_bias=False, activation="sigmoid", name="obstacles-size")(x)
-        # c3 = Dense(2, use_bias=False, activation="tanh", name="obstacles-coord")(x)
-        # outputs.append(c1)
-        # outputs.append(c2)
-        # outputs.append(c3)
+        c1 = Dense(1, use_bias=False, activation="sigmoid", name="obstacles")(x)
+        c2 = Dense(1, use_bias=False, activation="sigmoid", name="obstacles-size")(x)
+        c3 = Dense(2, use_bias=False, activation="tanh", name="obstacles-coord")(x)
+        outputs.append(c1)
+        outputs.append(c2)
+        outputs.append(c3)
 
         y1 = Dense(1, use_bias=False, activation="tanh", name="steering.0")(x)
         y2 = Dense(1, use_bias=False, activation="tanh", name="steering.5")(x)
@@ -307,15 +309,49 @@ def separable_model():
 
         z = Dense(3, use_bias=False, activation="softmax", name="zone")(x)
         outputs.append(z)
+
+        # decoder
+        # (3, 8, 256)
+
+        w = SeparableConv2D(256, 3, strides=1, use_bias=False, padding="same")(w)
+        w = Activation("relu")(w)
+        w = ZeroPadding2D(padding=((1, 0), (1, 0)))(w)
+        w = UpSampling2D(size=(2, 2))(w)
+        # (8, 18, 256)
+
+        w = SeparableConv2D(192, 3, strides=1, use_bias=False, padding="same")(w)
+        w = Activation("relu")(w)
+        w = ZeroPadding2D(padding=((1, 1), (1, 1)))(w)
+        w = UpSampling2D(size=(2, 2))(w)
+        # (20, 40, 192)
+
+        w = SeparableConv2D(96, 5, strides=1, use_bias=False, padding="same")(w)
+        w = Activation("relu")(w)
+        w = UpSampling2D(size=(2, 2))(w)
+        # (40, 80, 96)
+
+        w = SeparableConv2D(48, 5, strides=1, use_bias=False, padding="same")(w)
+        w = Activation("relu")(w)
+        w = UpSampling2D(size=(2, 2))(w)
+        # (80, 160, 48)
+
+        w = SeparableConv2D(24, 5, strides=1, use_bias=False, padding="same")(w)
+        w = Activation("relu")(w)
+
+        w = Conv2D(3, 3, strides=1, use_bias=False, padding="same")(w)
+        w = Activation("tanh")(w)
+        w = ZeroPadding2D(padding=((20, 20), (0, 0)))(w)
+        w = Lambda(lambda x: (x + 1) * 0.5, name="image.0")(w)
+        outputs.append(w)
 
         # Create the model
         model = Model(inputs=inputs, outputs=outputs)
 
         # Compile it
         model.compile(
             optimizer=Adam(),
-            loss=["mse", "mse", "mse", "mse", "mse", "mse", "categorical_crossentropy"],
-            loss_weights=[1, 1, 1, 1, 1, 1, 0.75],
+            loss=["mse", "mse", "mse", "mse", "mse", "mse", "categorical_crossentropy", "mse"],
+            loss_weights=[1, 1, 1, 1, 1, 1, 0.75, 1],
         )
 
         logging.info(f"created separable model with {get_flops(model)} FLOPS")

autopylot/models/train.py

@@ -62,7 +62,6 @@ def train(
         train_split=settings.TRAIN_SPLITS,
         shuffle=settings.TRAIN_SHUFFLE,
         verbose=settings.TRAIN_VERBOSE,
-        do_save=True,
         additionnal_funcs=[],
     ):
         """Trains the model on the given dataset.
@@ -135,7 +134,7 @@ def train(
             validation_data=test_generator,
             validation_steps=1,
             epochs=settings.TRAIN_EPOCHS,
-            workers=4,
+            workers=8,
         )
 
         if settings.MODEL_SAVE_SETTINGS:
@@ -147,4 +146,10 @@ def train(
                 "shuffle": shuffle,
                 "verbose": verbose,
             }
+
+        # save alternative version without decoder
+        encoder = utils.remove_decoder(self.model)
+        if encoder is not None:
+            utils.save_model(encoder, f"{self.name}_enc", model_info=self.model_info)
+
         utils.save_model(self.model, self.name, model_info=self.model_info)

autopylot/models/utils.py

@@ -144,11 +144,38 @@ def get_clean_layer_name(name):
     Returns:
         string: the filtered layer name.
     """
+
+    if name.endswith("_output"):
+        name = name.split("_output")[0]
+
     delimit_chars = ":_/"
     for char in delimit_chars:
         name = name.split(char)[0]
+
     return name
 
+def remove_decoder(model):
+    """Remove the decoder from the model.
+    removes the "image" output layer and rebuild the model.
+
+    Args:
+        model (Model): the model.
+
+    Returns:
+        Model: the model without the decoder. OR None if no changes
+    """
+
+    new_outputs = []
+    outputs = model.outputs
+    for output in outputs:
+        if "image" not in output.name:
+            new_outputs.append(output)
+
+    if len(new_outputs) == 0:
+        raise ValueError("No output left after removing decoder")
+
+    if len(new_outputs) != len(outputs):
+        return Model(model.inputs, new_outputs)
 
 def predict_decorator(func, inputs, outputs):
     """Decorate the model.predict function.

autopylot/tests/test_io.py

@@ -34,45 +34,52 @@ def test_create_directory():
 
 def test_load_image_none():
     """Testing if the image was loaded."""
-    image = io.load_image(os.getcwd() + "\\testing_io\\test.png")
-    assert image is None, "should not be None, the image doens't exist."
+    path = os.path.join(os.getcwd(), "testing_io", "test.png")
+    image = io.load_image(path)
+    assert image is None, "should be None, the image doens't exist."
 
 
 def test_save_image():
     """Testing the saving of an image."""
     image = np.zeros((2, 2, 3), dtype=np.uint8)
-    save = io.save_image(os.getcwd() + "\\testing_io\\test.png", image)
+    path = os.path.join(os.getcwd(), "testing_io", "test.png")
+    save = io.save_image(path, image)
     assert save is True, "Image not saved."
 
 
 def test_load_image():
     """Testing the loading of the image."""
-    image = io.load_image(os.getcwd() + "\\testing_io\\test.png")
+    path = os.path.join(os.getcwd(), "testing_io", "test.png")
+    image = io.load_image(path)
     assert image.shape == (2, 2, 3)
 
 
 def test_load_json_none():
     """Testing the loading of a non existing json (should raise an error)."""
     with pytest.raises(Exception):
-        io.load_json(os.getcwd() + "\\testing_io\\test.json")
+        path = os.path.join(os.getcwd(), "testing_io", "test.json")
+        io.load_json(path)
 
 
 def test_save_json():
     """Testing the saving of a dictionnary into a .json file."""
     data = {"test": "this is a test"}
-    save = io.save_json(os.getcwd() + "\\testing_io\\test.json", data)
+    path = os.path.join(os.getcwd(), "testing_io", "test.json")
+    save = io.save_json(path, data)
     assert save is True, "json not saved."
 
 
 def test_load_json():
     """Testing the loading of the .json file."""
-    data = io.load_json(os.getcwd() + "\\testing_io\\test.json")
+    path = os.path.join(os.getcwd(), "testing_io", "test.json")
+    data = io.load_json(path)
     assert data == {"test": "this is a test"}
 
 
 def test_load_image_data():
     """Testing the loading of both image and .json file."""
-    image_data = io.load_image_data(os.getcwd() + "\\testing_io\\test.json")
+    path = os.path.join(os.getcwd(), "testing_io", "test.json")
+    image_data = io.load_image_data(path)
     image = image_data["image"]
     del image_data["image"]
     assert image.shape == (2, 2, 3) and image_data == {"test": "this is a test"}
@@ -85,8 +92,9 @@ def test_save_image_data():
         "image": np.zeros((2, 2, 3), dtype=np.uint8),
     }
 
-    io.save_image_data(image_data, os.getcwd() + "\\testing_io\\test2.json")
-    image_data_copy = io.load_image_data(os.getcwd() + "\\testing_io\\test2.json")
+    path = os.path.join(os.getcwd(), "testing_io", "test2.json")
+    io.save_image_data(image_data, path)
+    image_data_copy = io.load_image_data(path)
 
     assert (
         image_data["image"].shape == image_data_copy["image"].shape

autopylot/utils/vis.py

@@ -41,7 +41,7 @@ def vis_line_scalar(
     return vis_image
 
 
-def vis_steering(image_data, image_key="image"):
+def vis_steering(image_data, image_key="image", color=(0, 0, 255)):
     """Visualize the steering scalar.
 
     Args:
@@ -57,12 +57,12 @@ def vis_steering(image_data, image_key="image"):
         pos=(0.5, 0),
         length=(0, 0.25),
         fact=(0.2, 0),
-        color=(0, 0, 255),
+        color=color,
         thickness=2,
     )
 
 
-def vis_throttle(image_data, image_key="image"):
+def vis_throttle(image_data, image_key="image", color=(0, 0, 255)):
     """Visualize the throttle scalar.
 
     Args:
@@ -73,7 +73,6 @@ def vis_throttle(image_data, image_key="image"):
         np.array: modified image with the drawn visualization.
     """
     throttle = image_data["throttle"]
-    color = (0, 0, 255)
     if throttle < 0:
         color = (255, 0, 0)
         throttle *= -1
@@ -215,7 +214,7 @@ def vis_obstacles(image_data, image_key="image", x_minmax=(-1, 1), y_minmax=(-1,
     return image_data[image_key]
 
 
-def vis_all(image_data, image_key="image"):
+def vis_all(image_data, image_key="image", color=(0, 0, 255)):
     """Visualize every data present in the image_data dictionary.
 
     Args: