core: Clamp min coordinates in color_transform

The current behavior assumes the max value can be larger than the
target size, but that the min can't.

This reasoning seems faulty, since one could pass (min:100,max:150)
to a target of size 50, in which case both min and max are larger,
but only max would get clamped.

We now clamp both min and max to correctly enforce invariants.

Author: crumblingstatue

core/src/bitmap/operations.rs

@@ -494,6 +494,9 @@ pub fn color_transform<'gc>(
         return;
     }
 
+    // Clamp both min and max coordinates to target size to enforce invariants
+    let x_min = x_min.min(target.width());
+    let y_min = y_min.min(target.height());
     let x_max = x_max.min(target.width());
     let y_max = y_max.min(target.height());
 

tests/tests/swfs/avm2/bitmapdata_colortransform_oob/Test.as

@@ -0,0 +1,25 @@
+package
+{
+   import flash.display.MovieClip;
+
+   public class Test extends MovieClip
+   {
+      public function Test()
+      {
+         super();
+      }
+   }
+}
+
+import flash.display.BitmapData;
+import flash.geom.ColorTransform;
+import flash.geom.Rectangle;
+
+var bitmap:BitmapData = new BitmapData(8,8,false,0);
+
+// Try to do a colorTransform with an out of bounds Rectangle.
+// This doesn't crash FP
+bitmap.colorTransform(new Rectangle(9,9,10,10),new ColorTransform(1,1,1,1,1,1,1,1));
+trace("// bitmap.rect (sanity check)");
+trace(bitmap.rect.toString());
+

tests/tests/swfs/avm2/bitmapdata_colortransform_oob/output.txt

@@ -0,0 +1,2 @@
+// bitmap.rect (sanity check)
+(x=0, y=0, w=8, h=8)

tests/tests/swfs/avm2/bitmapdata_colortransform_oob/test.swf

@@ -0,0 +1 @@
+num_frames = 1