[WIP] DL4J toSameDiff conversion method

deeplearning4j/deeplearning4j-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java

@@ -23,7 +23,9 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
+import org.junit.rules.DisableOnDebug;
 import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
 import org.nd4j.common.base.Preconditions;
 import org.nd4j.common.config.ND4JSystemProperties;
@@ -48,7 +50,7 @@ public abstract class BaseDL4JTest {
     @Rule
     public TestName name = new TestName();
     @Rule
-    public Timeout timeout = Timeout.millis(getTimeoutMilliseconds());
+    public TestRule timeout = new DisableOnDebug(Timeout.millis(getTimeoutMilliseconds()));
 
     protected long startTime;
     protected int threadCountBefore;

deeplearning4j/deeplearning4j-core/pom.xml

@@ -22,6 +22,7 @@
         <artifactId>deeplearning4j-parent</artifactId>
         <version>1.0.0-SNAPSHOT</version>
     </parent>
+
     <dependencyManagement>
         <dependencies>
             <dependency>
@@ -166,6 +167,46 @@
         </dependency>
     </dependencies>
 
+    <build>
+        <extensions>
+            <extension>
+                <groupId>org.apache.maven.wagon</groupId>
+                <artifactId>wagon-http</artifactId>
+                <version>2.9</version>
+            </extension>
+            <extension>
+                <groupId>org.kuali.maven.wagons</groupId>
+                <artifactId>maven-s3-wagon</artifactId>
+                <version>1.2.1</version>
+            </extension>
+        </extensions>
+
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <inherited>true</inherited>
+                    <configuration>
+                        <useSystemClassLoader>true</useSystemClassLoader>
+                        <useManifestOnlyJar>false</useManifestOnlyJar>
+                        <argLine>-Ddtype=float -Dfile.encoding=UTF-8 -Xmx8g</argLine>
+                        <includes>
+                            <!-- Default setting only runs tests that start/end with "Test" -->
+                            <include>*.java</include>
+                            <include>**/*.java</include>
+                        </includes>
+                        <properties>
+                            <property>
+                                <name>listener</name>
+                                <value>org.deeplearning4j.samediff.ToSameDiffTests</value>
+                            </property>
+                        </properties>
+                    </configuration>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+
     <profiles>
         <profile>
             <id>test-nd4j-native</id>

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/TestUtils.java

@@ -16,22 +16,53 @@
 
 package org.deeplearning4j;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import lombok.NonNull;
 import org.apache.commons.compress.utils.IOUtils;
 import org.deeplearning4j.nn.api.Layer;
 import org.deeplearning4j.nn.conf.ComputationGraphConfiguration;
 import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
 import org.deeplearning4j.nn.conf.RNNFormat;
+import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.BaseLayer;
 import org.deeplearning4j.nn.conf.layers.samediff.AbstractSameDiffLayer;
 import org.deeplearning4j.nn.graph.ComputationGraph;
+import org.deeplearning4j.nn.graph.vertex.GraphVertex;
+import org.deeplearning4j.nn.layers.BaseOutputLayer;
+import org.deeplearning4j.nn.layers.LossLayer;
 import org.deeplearning4j.nn.layers.convolution.ConvolutionLayer;
 import org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingLayer;
 import org.deeplearning4j.nn.layers.normalization.BatchNormalization;
 import org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization;
 import org.deeplearning4j.nn.layers.recurrent.LSTM;
 import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
 import org.deeplearning4j.util.ModelSerializer;
+import org.nd4j.autodiff.samediff.SDVariable;
+import org.nd4j.autodiff.samediff.SameDiff;
 import org.nd4j.common.base.Preconditions;
+import org.nd4j.common.primitives.Pair;
 import org.nd4j.linalg.api.buffer.DataType;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution;
@@ -40,14 +71,7 @@
 import org.nd4j.linalg.learning.regularization.L2Regularization;
 import org.nd4j.linalg.learning.regularization.Regularization;
 import org.nd4j.linalg.learning.regularization.WeightDecay;
-
-import java.io.*;
-import java.lang.reflect.Field;
-import java.util.List;
-import java.util.Random;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import org.nd4j.linalg.lossfunctions.ILossFunction;
 
 public class TestUtils {
 

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java

@@ -28,6 +28,8 @@
 import org.deeplearning4j.nn.graph.ComputationGraph;
 import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
 import org.deeplearning4j.nn.weights.WeightInit;
+import org.deeplearning4j.samediff.TestToSameDiff;
+import org.deeplearning4j.samediff.ToSameDiffTests;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -106,6 +108,7 @@ public void testSelfAttentionLayer() {
                     boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in)
                             .labels(labels).inputMask(inMask).subset(true).maxPerParam(100));
                     assertTrue(name, gradOK);
+                    ToSameDiffTests.testToSameDiff(net, in, labels);
                 }
             }
         }
@@ -167,6 +170,7 @@ public void testLearnedSelfAttentionLayer() {
                     boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in)
                             .labels(labels).inputMask(inMask).subset(true).maxPerParam(100));
                     assertTrue(name, gradOK);
+                    ToSameDiffTests.testToSameDiff(net, in, labels);
                 }
             }
         }
@@ -322,6 +326,8 @@ public void testRecurrentAttentionLayer() {
                 boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in)
                         .labels(labels).inputMask(inMask).subset(true).maxPerParam(100));
                 assertTrue(name, gradOK);
+
+                ToSameDiffTests.testToSameDiff(net, in, labels);
             }
         }
     }
@@ -385,6 +391,7 @@ public void testAttentionVertex() {
                     boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{in})
                             .labels(new INDArray[]{labels}).inputMask(inMask != null ? new INDArray[]{inMask} : null).subset(true).maxPerParam(100));
                     assertTrue(name, gradOK);
+                    ToSameDiffTests.testToSameDiff(net, in, labels);
                 }
             }
         }
@@ -447,6 +454,7 @@ public void testAttentionVertexSameInput() {
                     boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{in})
                             .labels(new INDArray[]{labels}).inputMask(inMask != null ? new INDArray[]{inMask} : null));
                     assertTrue(name, gradOK);
+                    ToSameDiffTests.testToSameDiff(net, in, labels);
                 }
             }
         }

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java

@@ -30,20 +30,18 @@
 import org.deeplearning4j.nn.graph.ComputationGraph;
 import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
 import org.deeplearning4j.nn.weights.WeightInit;
+import org.deeplearning4j.samediff.ToSameDiffTests;
 import org.junit.Test;
 import org.nd4j.linalg.activations.Activation;
 import org.nd4j.linalg.api.buffer.DataType;
 import org.nd4j.linalg.api.ndarray.INDArray;
-import org.nd4j.linalg.api.ops.executioner.OpExecutioner;
 import org.nd4j.linalg.dataset.DataSet;
 import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
 import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization;
 import org.nd4j.linalg.dataset.api.preprocessor.NormalizerMinMaxScaler;
 import org.nd4j.linalg.factory.Nd4j;
 import org.nd4j.linalg.learning.config.NoOp;
 import org.nd4j.linalg.lossfunctions.LossFunctions;
-import org.nd4j.linalg.profiler.OpProfiler;
-import org.nd4j.linalg.profiler.ProfilerConfig;
 
 import java.util.Arrays;
 import java.util.HashSet;
@@ -104,6 +102,7 @@ public void testGradient2dSimple() {
                     .labels(labels).excludeParams(excludeParams));
 
             assertTrue(gradOK);
+            ToSameDiffTests.testToSameDiff(mln, input, labels);
             TestUtils.testModelSerialization(mln);
         }
     }
@@ -149,6 +148,7 @@ public void testGradientCnnSimple() {
                     .labels(labels).excludeParams(excludeParams));
 
             assertTrue(gradOK);
+            ToSameDiffTests.testToSameDiff(mln, input, labels);
             TestUtils.testModelSerialization(mln);
         }
     }
@@ -251,6 +251,7 @@ public void testGradientBNWithCNNandSubsampling() {
                                     .labels(labels).excludeParams(excludeParams).subset(true).maxPerParam(25)); //Most params are in output layer, only these should be skipped with this threshold
 
                             assertTrue(gradOK);
+                            ToSameDiffTests.testToSameDiff(mln, input, labels);
                             TestUtils.testModelSerialization(mln);
                         }
                     }
@@ -355,6 +356,7 @@ public void testGradientDense() {
                                     .labels(labels).excludeParams(excludeParams));
 
                             assertTrue(gradOK);
+                            ToSameDiffTests.testToSameDiff(mln, input, labels);
                             TestUtils.testModelSerialization(mln);
                         }
                     }
@@ -399,6 +401,7 @@ public void testGradient2dFixedGammaBeta() {
                     .labels(labels).excludeParams(excludeParams));
 
             assertTrue(gradOK);
+            ToSameDiffTests.testToSameDiff(mln, input, labels);
             TestUtils.testModelSerialization(mln);
         }
     }
@@ -444,6 +447,7 @@ public void testGradientCnnFixedGammaBeta() {
                     .labels(labels).excludeParams(excludeParams));
 
             assertTrue(gradOK);
+            ToSameDiffTests.testToSameDiff(mln, input, labels);
             TestUtils.testModelSerialization(mln);
         }
     }
@@ -489,6 +493,7 @@ public void testBatchNormCompGraphSimple() {
 
             assertTrue(gradOK);
             TestUtils.testModelSerialization(net);
+            ToSameDiffTests.testToSameDiff(net, input, labels);
         }
     }
 
@@ -587,6 +592,7 @@ public void testGradientBNWithCNNandSubsamplingCompGraph() {
 
                         assertTrue(gradOK);
                         TestUtils.testModelSerialization(net);
+                        ToSameDiffTests.testToSameDiff(net, input, labels);
                     }
                 }
             }

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java

@@ -27,8 +27,8 @@
 import org.deeplearning4j.nn.conf.layers.*;
 import org.deeplearning4j.nn.conf.layers.convolutional.Cropping1D;
 import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
+import org.deeplearning4j.samediff.ToSameDiffTests;
 import org.deeplearning4j.util.Convolution1DUtils;
-import org.deeplearning4j.util.ConvolutionUtils;
 import org.junit.Test;
 import org.nd4j.linalg.activations.Activation;
 import org.nd4j.linalg.api.buffer.DataType;
@@ -122,6 +122,7 @@ public void testCnn1DWithLocallyConnected1D() {
 
                     assertTrue(msg, gradOK);
 
+                    ToSameDiffTests.testToSameDiff(net, input, labels);
                     TestUtils.testModelSerialization(net);
                 }
 
@@ -202,6 +203,7 @@ public void testCnn1DWithCropping1D() {
 
                         assertTrue(msg, gradOK);
 
+                        ToSameDiffTests.testToSameDiff(net, input, labels);
                         TestUtils.testModelSerialization(net);
                     }
                 }
@@ -285,6 +287,7 @@ public void testCnn1DWithZeroPadding1D() {
                                 DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
 
                         assertTrue(msg, gradOK);
+                        ToSameDiffTests.testToSameDiff(net, input, labels);
                         TestUtils.testModelSerialization(net);
                     }
                 }
@@ -362,6 +365,7 @@ public void testCnn1DWithSubsampling1D() {
                                 DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
 
                         assertTrue(msg, gradOK);
+                        ToSameDiffTests.testToSameDiff(net, input, labels);
                         TestUtils.testModelSerialization(net);
                     }
                 }
@@ -423,6 +427,7 @@ public void testCnn1dWithMasking(){
                             .labels(label).inputMask(fm));
 
                     assertTrue(s, gradOK);
+                    ToSameDiffTests.testToSameDiff(net, f, label);
                     TestUtils.testModelSerialization(net);
 
                     //TODO also check that masked step values don't impact forward pass, score or gradients
@@ -518,6 +523,7 @@ public void testCnn1Causal() {
                     .labels(label).inputMask(fm));
 
             assertTrue(s, gradOK);
+            ToSameDiffTests.testToSameDiff(net, f, label);
             TestUtils.testModelSerialization(net);
         }
     }

deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java

@@ -29,9 +29,9 @@
 import org.deeplearning4j.nn.conf.preprocessor.Cnn3DToFeedForwardPreProcessor;
 import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
 import org.deeplearning4j.nn.weights.WeightInit;
+import org.deeplearning4j.samediff.ToSameDiffTests;
 import org.junit.Test;
 import org.nd4j.linalg.activations.Activation;
-import org.nd4j.linalg.api.buffer.DataBuffer;
 import org.nd4j.linalg.api.buffer.DataType;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.factory.Nd4j;
@@ -159,6 +159,7 @@ public void testCnn3DPlain() {
 
                                             assertTrue(msg, gradOK);
 
+                                            ToSameDiffTests.testToSameDiff(net, input, labels);
                                             TestUtils.testModelSerialization(net);
                                         }
                                     }
@@ -262,6 +263,7 @@ public void testCnn3DZeroPadding() {
 
                     assertTrue(msg, gradOK);
 
+                    ToSameDiffTests.testToSameDiff(net, input, labels);
                     TestUtils.testModelSerialization(net);
                 }
 
@@ -352,6 +354,7 @@ public void testCnn3DPooling() {
 
                             assertTrue(msg, gradOK);
 
+                            ToSameDiffTests.testToSameDiff(net, input, labels);
                             TestUtils.testModelSerialization(net);
                         }
                     }
@@ -442,6 +445,7 @@ public void testCnn3DUpsampling() {
 
                         assertTrue(msg, gradOK);
 
+                        ToSameDiffTests.testToSameDiff(net, input, labels);
                         TestUtils.testModelSerialization(net);
                     }
                 }
@@ -541,6 +545,7 @@ public void testCnn3DCropping() {
 
                     assertTrue(msg, gradOK);
 
+                    ToSameDiffTests.testToSameDiff(net, input, labels);
                     TestUtils.testModelSerialization(net);
                 }
 
@@ -632,6 +637,7 @@ public void testDeconv3d() {
 
             assertTrue(msg, gradOK);
 
+            ToSameDiffTests.testToSameDiff(net, input, labels);
             TestUtils.testModelSerialization(net);
         }
     }