wip border stripes fix

from #74

sets border = 'smart' by default

Author: kabachuha

scripts/deforum_helpers/animation.py

@@ -247,12 +247,28 @@ def anim_frame_warp_2d(prev_img_cv2, args, anim_args, keys, frame_idx):
     else:
         xform = np.matmul(rot_mat, trans_mat)
 
-    return cv2.warpPerspective(
-        prev_img_cv2,
-        xform,
-        (prev_img_cv2.shape[1], prev_img_cv2.shape[0]),
-        borderMode=cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE
-    )
+    borderMode = cv2.BORDER_CONSTANT #zeros
+
+    if anim_args.border == 'wrap':
+        borderMode = cv2.BORDER_WRAP
+    elif anim_args.border == 'replicate':
+        borderMode = cv2.BORDER_REPLICATE
+
+    if borderMode == 'smart':
+        return cv2.warpPerspective(
+            prev_img_cv2,
+            xform,
+            (prev_img_cv2.shape[1], prev_img_cv2.shape[0]),
+            borderMode=borderMode,
+            borderValue=(0, 0, 0,),
+        )
+    else:
+        return cv2.warpPerspective(
+            prev_img_cv2,
+            xform,
+            (prev_img_cv2.shape[1], prev_img_cv2.shape[0]),
+            borderMode=borderMode,
+        )
 
 def anim_frame_warp_3d(device, prev_img_cv2, depth, anim_args, keys, frame_idx):
     TRANSLATION_SCALE = 1.0/200.0 # matches Disco
@@ -305,7 +321,7 @@ def transform_image_3d(device, prev_img_cv2, depth_tensor, rot_mat, translate, a
         image_tensor.add(1/512 - 0.0001).unsqueeze(0), 
         offset_coords_2d, 
         mode=anim_args.sampling_mode, 
-        padding_mode=anim_args.padding_mode, 
+        padding_mode=anim_args.padding_mode if anim_args.border is not 'smart' else 'zeros', # border overrides padding_mode without changing the settings saved
         align_corners=False
     )
 

scripts/deforum_helpers/args.py

@@ -22,7 +22,7 @@ def DeforumAnimArgs():
     #@markdown ####**Animation:**
     animation_mode = '2D' #@param ['None', '2D', '3D', 'Video Input', 'Interpolation'] {type:'string'}
     max_frames = 120 #@param {type:"number"}
-    border = 'replicate' #@param ['wrap', 'replicate', 'smart'] {type:'string'}
+    border = 'smart' #@param ['wrap', 'replicate', 'smart'] {type:'string'}
 
     #@markdown ####**Motion Parameters:**
     angle = "0:(0)"#@param {type:"string"}
@@ -327,7 +327,7 @@ def show_vid():
         with gr.Row():
             animation_mode = gr.Dropdown(label="animation_mode", choices=['2D', '3D', 'Video Input', 'Interpolation'], value=da.animation_mode, type="value", elem_id="animation_mode", interactive=True)
             max_frames = gr.Number(label="max_frames", value=da.max_frames, interactive=True, precision=0)
-            border = gr.Dropdown(label="border", choices=['replicate', 'wrap'], value=da.border, type="value", elem_id="border", interactive=True)
+            border = gr.Dropdown(label="border", choices=['replicate', 'wrap', 'smart'], value=da.border, type="value", elem_id="border", interactive=True)
 
 
         i10 = gr.HTML("<p style=\"margin-bottom:0.75em\">Motion parameters:</p>")

scripts/deforum_helpers/generate.py

@@ -160,6 +160,39 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
         open_cv_image = sample_to_cv2(args.init_sample)
         img = cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2RGB)
         init_image = Image.fromarray(img)
+
+        if anim_args.border == 'smart':
+            
+            # Inpaint changed parts of the image
+            # that's, to say, zeros we got after the transformations
+
+            mask_image = init_image.convert('L')
+            for x in range(mask_image.width):
+                for y in range(mask_image.height):
+                    if mask_image.getpixel((x,y)) < 1:
+                        mask_image.putpixel((x,y), 255)
+                    else:
+                        mask_image.putpixel((x,y), 0)
+            
+            mask = prepare_mask(mask_image, 
+                                    (args.W, args.H), 
+                                    args.mask_contrast_adjust, 
+                                    args.mask_brightness_adjust, 
+                                    invert_mask=False)
+
+            p.inpainting_fill = args.fill # need to come up with better name. 
+            p.inpaint_full_res= args.full_res_mask 
+            p.inpaint_full_res_padding = args.full_res_mask_padding 
+
+            p.init_images = [init_image]
+            p.image_mask = mask
+
+            processed = processing.process_images(p)
+
+            init_image = processed.images[0].convert('RGB') 
+            p.image_mask = None
+            mask_image = None
+            processed = None
     elif args.use_init and args.init_image != None and args.init_image != '':
         init_image, mask_image = load_img(args.init_image, 
                                           shape=(args.W, args.H),  
@@ -218,7 +251,7 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
         assert not ( (args.use_mask and args.overlay_mask) and (args.init_sample is None and init_image is None)), "Need an init image when use_mask == True and overlay_mask == True"
 
         p.init_images = [init_image]
-        p.image_mask = mask
+        p.image_mask = mask_image
 
         print(f"seed={p.seed}; subseed={p.subseed}; subseed_strength={p.subseed_strength}; denoising_strength={p.denoising_strength}; steps={p.steps}; cfg_scale={p.cfg_scale}")
         processed = processing.process_images(p)