Return 0 or -1 from ImagingTransform()

src/_imaging.c

@@ -2041,10 +2041,16 @@ _resize(ImagingObject *self, PyObject *args) {
         a[5] = box[1];
 
         imOut = ImagingNewDirty(imIn->mode, xsize, ysize);
+        if (!imOut) {
+            return NULL;
+        }
 
-        imOut = ImagingTransform(
-            imOut, imIn, IMAGING_TRANSFORM_AFFINE, 0, 0, xsize, ysize, a, filter, 1
-        );
+        if (ImagingTransform(
+                imOut, imIn, IMAGING_TRANSFORM_AFFINE, 0, 0, xsize, ysize, a, filter, 1
+            )) {
+            ImagingDelete(imOut);
+            return NULL;
+        }
     } else {
         imOut = ImagingResample(imIn, xsize, ysize, filter, box);
     }
@@ -2167,13 +2173,13 @@ _transform(ImagingObject *self, PyObject *args) {
         return NULL;
     }
 
-    imOut = ImagingTransform(
+    int error = ImagingTransform(
         self->image, imagep->image, method, x0, y0, x1, y1, a, filter, fill
     );
 
     free(a);
 
-    if (!imOut) {
+    if (error) {
         return NULL;
     }
 

src/libImaging/Geometry.c

@@ -779,7 +779,7 @@ getfilter(Imaging im, int filterid) {
 
 /* transformation engines */
 
-Imaging
+int
 ImagingGenericTransform(
     Imaging imOut,
     Imaging imIn,
@@ -801,12 +801,14 @@ ImagingGenericTransform(
     double xx, yy;
 
     if (!imOut || !imIn || imIn->mode != imOut->mode) {
-        return (Imaging)ImagingError_ModeError();
+        ImagingError_ModeError();
+        return -1;
     }
 
     ImagingTransformFilter filter = getfilter(imIn, filterid);
     if (!filter) {
-        return (Imaging)ImagingError_ValueError("bad filter number");
+        ImagingError_ValueError("bad filter number");
+        return -1;
     }
 
     ImagingCopyPalette(imOut, imIn);
@@ -841,10 +843,10 @@ ImagingGenericTransform(
 
     ImagingSectionLeave(&cookie);
 
-    return imOut;
+    return 0;
 }
 
-static Imaging
+static int
 ImagingScaleAffine(
     Imaging imOut, Imaging imIn, int x0, int y0, int x1, int y1, double a[6], int fill
 ) {
@@ -858,7 +860,8 @@ ImagingScaleAffine(
     int *xintab;
 
     if (!imOut || !imIn || imIn->mode != imOut->mode) {
-        return (Imaging)ImagingError_ModeError();
+        ImagingError_ModeError();
+        return -1;
     }
 
     ImagingCopyPalette(imOut, imIn);
@@ -882,8 +885,8 @@ ImagingScaleAffine(
     /* malloc check ok, uses calloc for overflow */
     xintab = (int *)calloc(out_xsize, sizeof(int));
     if (!xintab) {
-        ImagingDelete(imOut);
-        return (Imaging)ImagingError_MemoryError();
+        ImagingError_MemoryError();
+        return -1;
     }
 
     xo = a[2] + a[0] * 0.5;
@@ -936,7 +939,7 @@ ImagingScaleAffine(
 
     free(xintab);
 
-    return imOut;
+    return 0;
 }
 
 static inline int
@@ -947,7 +950,7 @@ check_fixed(double a[6], int x, int y) {
     );
 }
 
-static inline Imaging
+static inline int
 affine_fixed(
     Imaging imOut,
     Imaging imIn,
@@ -1022,10 +1025,10 @@ affine_fixed(
 
 #undef AFFINE_TRANSFORM_FIXED
 
-    return imOut;
+    return 0;
 }
 
-Imaging
+int
 ImagingTransformAffine(
     Imaging imOut,
     Imaging imIn,
@@ -1048,7 +1051,8 @@ ImagingTransformAffine(
     double xo, yo;
 
     if (!imOut || !imIn || imIn->mode != imOut->mode) {
-        return (Imaging)ImagingError_ModeError();
+        ImagingError_ModeError();
+        return -1;
     }
 
     if (filterid || imIn->type == IMAGING_TYPE_I16) {
@@ -1131,10 +1135,25 @@ ImagingTransformAffine(
 
 #undef AFFINE_TRANSFORM
 
-    return imOut;
+    return 0;
 }
 
-Imaging
+/**
+ * Runs a transformation over the data from imIn, placing the result in imOut.
+ *
+ * @param imOut     Existing image to write into
+ * @param imIn      Source image to transform
+ * @param method    Which method to use
+ * @param x0        The left co-ordinate of the area to transform
+ * @param y0        The upper co-ordinate of the area to transform
+ * @param x1        The right co-ordinate of the area to transform
+ * @param y1        The lower co-ordinate of the area to transform
+ * @param a         An array of extra data for the transformation method
+ * @param filterid  Which filter to use
+ * @param fill      1 to fill the area with zeros
+ * @return          -1 in case of an error, 0 otherwise
+ */
+int
 ImagingTransform(
     Imaging imOut,
     Imaging imIn,
@@ -1162,7 +1181,8 @@ ImagingTransform(
             transform = quad_transform;
             break;
         default:
-            return (Imaging)ImagingError_ValueError("bad transform method");
+            ImagingError_ValueError("bad transform method");
+            return -1;
     }
 
     return ImagingGenericTransform(

src/libImaging/Imaging.h

@@ -389,7 +389,7 @@ extern Imaging
 ImagingResample(Imaging imIn, int xsize, int ysize, int filter, float box[4]);
 extern Imaging
 ImagingReduce(Imaging imIn, int xscale, int yscale, int box[4]);
-extern Imaging
+extern int
 ImagingTransform(
     Imaging imOut,
     Imaging imIn,