From cb33994f05fa359f17a677b01ea78fd1029a0c48 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 20:36:35 +0300
Subject: [PATCH 01/19] add border mode to args and docs

---
 javascript/deforum-hints.js     | 1 +
 scripts/deforum_helpers/args.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/javascript/deforum-hints.js b/javascript/deforum-hints.js
index de1e3a533..d26e62c7a 100644
--- a/javascript/deforum-hints.js
+++ b/javascript/deforum-hints.js
@@ -40,6 +40,7 @@ deforum_titles = {
     "border": "controls handling method of pixels to be generated when the image is smaller than the frame.",
         "wrap": "pulls pixels from the opposite edge of the image",
         "replicate": "repeats the edge of the pixels, and extends them. Animations with quick motion may yield lines where this border function was attempting to populate pixels into the empty space created.",
+        "smart": "makes a second 'inpaint' pass to fill empty space when zooming out or shifting the frame; useful to fix the 'border stripes' and make the 'sidespace' feel more open; overrides padding_mode",
     "angle": "2D operator to rotate canvas clockwise/anticlockwise in degrees per frame",
     "zoom": "2D operator that scales the canvas size, multiplicatively. [static = 1.0]",
     "translation_x": "2D & 3D operator to move canvas left/right in pixels per frame",
diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py
index 5174eaba2..cb3b412f0 100644
--- a/scripts/deforum_helpers/args.py
+++ b/scripts/deforum_helpers/args.py
@@ -22,7 +22,7 @@ def DeforumAnimArgs():
     #@markdown ####**Animation:**
     animation_mode = '2D' #@param ['None', '2D', '3D', 'Video Input', 'Interpolation'] {type:'string'}
     max_frames = 120 #@param {type:"number"}
-    border = 'replicate' #@param ['wrap', 'replicate'] {type:'string'}
+    border = 'replicate' #@param ['wrap', 'replicate', 'smart'] {type:'string'}
 
     #@markdown ####**Motion Parameters:**
     angle = "0:(0)"#@param {type:"string"}

From 267c0e6526707b05109a045b8dcbfa3ee4d72dde Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 20:51:38 +0300
Subject: [PATCH 02/19] wip border stripes fix

from #74

sets border = 'smart' by default
---
 scripts/deforum_helpers/animation.py | 30 ++++++++++++++++++------
 scripts/deforum_helpers/args.py      |  4 ++--
 scripts/deforum_helpers/generate.py  | 35 +++++++++++++++++++++++++++-
 3 files changed, 59 insertions(+), 10 deletions(-)

diff --git a/scripts/deforum_helpers/animation.py b/scripts/deforum_helpers/animation.py
index fbc45531f..ce559ad3a 100644
--- a/scripts/deforum_helpers/animation.py
+++ b/scripts/deforum_helpers/animation.py
@@ -247,12 +247,28 @@ def anim_frame_warp_2d(prev_img_cv2, args, anim_args, keys, frame_idx):
     else:
         xform = np.matmul(rot_mat, trans_mat)
 
-    return cv2.warpPerspective(
-        prev_img_cv2,
-        xform,
-        (prev_img_cv2.shape[1], prev_img_cv2.shape[0]),
-        borderMode=cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE
-    )
+    borderMode = cv2.BORDER_CONSTANT #zeros
+
+    if anim_args.border == 'wrap':
+        borderMode = cv2.BORDER_WRAP
+    elif anim_args.border == 'replicate':
+        borderMode = cv2.BORDER_REPLICATE
+
+    if borderMode == 'smart':
+        return cv2.warpPerspective(
+            prev_img_cv2,
+            xform,
+            (prev_img_cv2.shape[1], prev_img_cv2.shape[0]),
+            borderMode=borderMode,
+            borderValue=(0, 0, 0,),
+        )
+    else:
+        return cv2.warpPerspective(
+            prev_img_cv2,
+            xform,
+            (prev_img_cv2.shape[1], prev_img_cv2.shape[0]),
+            borderMode=borderMode,
+        )
 
 def anim_frame_warp_3d(device, prev_img_cv2, depth, anim_args, keys, frame_idx):
     TRANSLATION_SCALE = 1.0/200.0 # matches Disco
@@ -305,7 +321,7 @@ def transform_image_3d(device, prev_img_cv2, depth_tensor, rot_mat, translate, a
         image_tensor.add(1/512 - 0.0001).unsqueeze(0), 
         offset_coords_2d, 
         mode=anim_args.sampling_mode, 
-        padding_mode=anim_args.padding_mode, 
+        padding_mode=anim_args.padding_mode if anim_args.border is not 'smart' else 'zeros', # border overrides padding_mode without changing the settings saved
         align_corners=False
     )
 
diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py
index cb3b412f0..816bcbe45 100644
--- a/scripts/deforum_helpers/args.py
+++ b/scripts/deforum_helpers/args.py
@@ -22,7 +22,7 @@ def DeforumAnimArgs():
     #@markdown ####**Animation:**
     animation_mode = '2D' #@param ['None', '2D', '3D', 'Video Input', 'Interpolation'] {type:'string'}
     max_frames = 120 #@param {type:"number"}
-    border = 'replicate' #@param ['wrap', 'replicate', 'smart'] {type:'string'}
+    border = 'smart' #@param ['wrap', 'replicate', 'smart'] {type:'string'}
 
     #@markdown ####**Motion Parameters:**
     angle = "0:(0)"#@param {type:"string"}
@@ -327,7 +327,7 @@ def show_vid():
         with gr.Row():
             animation_mode = gr.Dropdown(label="animation_mode", choices=['2D', '3D', 'Video Input', 'Interpolation'], value=da.animation_mode, type="value", elem_id="animation_mode", interactive=True)
             max_frames = gr.Number(label="max_frames", value=da.max_frames, interactive=True, precision=0)
-            border = gr.Dropdown(label="border", choices=['replicate', 'wrap'], value=da.border, type="value", elem_id="border", interactive=True)
+            border = gr.Dropdown(label="border", choices=['replicate', 'wrap', 'smart'], value=da.border, type="value", elem_id="border", interactive=True)
         
         
         i10 = gr.HTML("<p style=\"margin-bottom:0.75em\">Motion parameters:</p>")
diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index e2e3bedbc..77d3fe387 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -160,6 +160,39 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
         open_cv_image = sample_to_cv2(args.init_sample)
         img = cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2RGB)
         init_image = Image.fromarray(img)
+
+        if anim_args.border == 'smart':
+            
+            # Inpaint changed parts of the image
+            # that's, to say, zeros we got after the transformations
+
+            mask_image = init_image.convert('L')
+            for x in range(mask_image.width):
+                for y in range(mask_image.height):
+                    if mask_image.getpixel((x,y)) < 1:
+                        mask_image.putpixel((x,y), 255)
+                    else:
+                        mask_image.putpixel((x,y), 0)
+            
+            mask = prepare_mask(mask_image, 
+                                    (args.W, args.H), 
+                                    args.mask_contrast_adjust, 
+                                    args.mask_brightness_adjust, 
+                                    invert_mask=False)
+
+            p.inpainting_fill = args.fill # need to come up with better name. 
+            p.inpaint_full_res= args.full_res_mask 
+            p.inpaint_full_res_padding = args.full_res_mask_padding 
+
+            p.init_images = [init_image]
+            p.image_mask = mask
+
+            processed = processing.process_images(p)
+
+            init_image = processed.images[0].convert('RGB') 
+            p.image_mask = None
+            mask_image = None
+            processed = None
     elif args.use_init and args.init_image != None and args.init_image != '':
         init_image, mask_image = load_img(args.init_image, 
                                           shape=(args.W, args.H),  
@@ -218,7 +251,7 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
         assert not ( (args.use_mask and args.overlay_mask) and (args.init_sample is None and init_image is None)), "Need an init image when use_mask == True and overlay_mask == True"
         
         p.init_images = [init_image]
-        p.image_mask = mask
+        p.image_mask = mask_image
 
         print(f"seed={p.seed}; subseed={p.subseed}; subseed_strength={p.subseed_strength}; denoising_strength={p.denoising_strength}; steps={p.steps}; cfg_scale={p.cfg_scale}")
         processed = processing.process_images(p)

From e9e44c717c28d843c0f886b1272d83b8c2776664 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 21:48:29 +0300
Subject: [PATCH 03/19] adjust tqdm for the 'smart' mode

---
 scripts/deforum_helpers/settings.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/deforum_helpers/settings.py b/scripts/deforum_helpers/settings.py
index 16c174be2..4012ea03f 100644
--- a/scripts/deforum_helpers/settings.py
+++ b/scripts/deforum_helpers/settings.py
@@ -183,6 +183,9 @@ def reset(self):
                 deforum_total += self._args.steps
                 had_first = True
             else:
+                #duplicate steps count in smart border mode
+                if self._anim_args.border == 'smart':
+                    deforum_total += int(ceil(self._args.steps * (1-strength)))
                 deforum_total += int(ceil(self._args.steps * (1-strength)))
 
             if turbo_steps > 1:

From c64297e9ef5b3b3e7353010d04c6f1e1f9d8ee87 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 22:17:03 +0300
Subject: [PATCH 04/19] make selectable smart border fill mode

---
 scripts/deforum.py                  | 12 ++++++------
 scripts/deforum_helpers/args.py     | 12 ++++++++----
 scripts/deforum_helpers/settings.py | 20 ++++++++++++++++----
 3 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/scripts/deforum.py b/scripts/deforum.py
index 13d5e47d1..00a9336e3 100644
--- a/scripts/deforum.py
+++ b/scripts/deforum.py
@@ -55,11 +55,11 @@ def ui(self, is_img2img):
     def show(self, is_img2img):
         return is_img2img
     
-    def run(self, p, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest, parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38):
+    def run(self, p, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest, parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38):
         print('Deforum script for 2D, pseudo-2D and 3D animations')
         print('v0.5-webui-beta')
         
-        root, args, anim_args, video_args, parseq_args = deforum_args.process_args(self, p, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest,  parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38)
+        root, args, anim_args, video_args, parseq_args = deforum_args.process_args(self, p, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest,  parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38)
 
         # Install numexpr as it's the thing most people are having problems with
         from launch import is_installed, run_pip
@@ -234,7 +234,7 @@ def run(self, p, override_settings_with_file, custom_settings_file, animation_mo
         root.initial_info += "Only the first frame is shown in webui not to clutter the memory"
         return Processed(p, [root.first_frame], root.initial_seed, root.initial_info)
 
-def run_deforum(override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest, parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38):
+def run_deforum(override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest, parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38):
 
     p = StableDiffusionProcessingImg2Img(
         sd_model=shared.sd_model,
@@ -245,7 +245,7 @@ def run_deforum(override_settings_with_file, custom_settings_file, animation_mod
 
     override_these_with_webui = False
 
-    processed = DeforumScript.run(None, p, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest, parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38)
+    processed = DeforumScript.run(None, p, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest, parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38)
     
     if processed is None:
         processed = process_images(p)
@@ -362,7 +362,7 @@ def close_vid():
 
 
         ds = SimpleNamespace(**components)
-        component_list = [ds.override_settings_with_file, ds.custom_settings_file, ds.animation_mode, ds.max_frames, ds.border, ds.angle, ds.zoom, ds.translation_x, ds.translation_y, ds.translation_z, ds.rotation_3d_x, ds.rotation_3d_y, ds.rotation_3d_z, ds.flip_2d_perspective, ds.perspective_flip_theta, ds.perspective_flip_phi, ds.perspective_flip_gamma, ds.perspective_flip_fv, ds.noise_schedule, ds.strength_schedule, ds.contrast_schedule, ds.cfg_scale_schedule, ds.fov_schedule, ds.near_schedule, ds.far_schedule, ds.seed_schedule, ds.kernel_schedule, ds.sigma_schedule, ds.amount_schedule, ds.threshold_schedule, ds.histogram_matching, ds.color_coherence, ds.diffusion_cadence, ds.use_depth_warping, ds.midas_weight, ds.near_plane, ds.far_plane, ds.fov, ds.padding_mode, ds.sampling_mode, ds.save_depth_maps, ds.video_init_path, ds.extract_nth_frame, ds.overwrite_extracted_frames, ds.use_mask_video, ds.video_mask_path, ds.interpolate_key_frames, ds.interpolate_x_frames, ds.resume_from_timestring, ds.resume_timestring, ds.prompts, ds.animation_prompts, ds.W, ds.H, ds.restore_faces, ds.tiling, ds.enable_hr, ds.firstphase_width, ds.firstphase_height, ds.seed, ds.sampler, ds.seed_enable_extras, ds.subseed, ds.subseed_strength, ds.seed_resize_from_w, ds.seed_resize_from_h, ds.steps, ds.ddim_eta, ds.n_batch, ds.make_grid, ds.grid_rows, ds.save_settings, ds.save_samples, ds.display_samples, ds.save_sample_per_step, ds.show_sample_per_step, ds.override_these_with_webui, ds.batch_name, ds.filename_format, ds.seed_behavior, ds.use_init, ds.from_img2img_instead_of_link, ds.strength_0_no_init, ds.strength, ds.init_image, ds.use_mask, ds.use_alpha_as_mask, ds.invert_mask, ds.overlay_mask, ds.mask_file, ds.mask_contrast_adjust, ds.mask_brightness_adjust, ds.mask_overlay_blur, ds.fill, ds.full_res_mask, ds.full_res_mask_padding, ds.reroll_blank_frames, ds.skip_video_for_run_all, ds.fps, ds.output_format, ds.ffmpeg_location, ds.add_soundtrack, ds.soundtrack_path, ds.use_manual_settings, ds.render_steps, ds.max_video_frames, ds.path_name_modifier, ds.image_path, ds.mp4_path, ds.parseq_manifest, ds.parseq_use_deltas, ds.i1, ds.i2, ds.i3, ds.i4, ds.i5, ds.i6, ds.i7, ds.i8, ds.i9, ds.i10, ds.i11, ds.i12, ds.i13, ds.i14, ds.i15, ds.i16, ds.i17, ds.i18, ds.i19, ds.i20, ds.i21, ds.i22, ds.i23, ds.i24, ds.i25, ds.i26, ds.i27, ds.i28, ds.i29, ds.i30, ds.i31, ds.i32, ds.i33, ds.i34, ds.i35, ds.i36, ds.i37, ds.i38]
+        component_list = [ds.override_settings_with_file, ds.custom_settings_file, ds.animation_mode, ds.max_frames, ds.border, ds.angle, ds.zoom, ds.translation_x, ds.translation_y, ds.translation_z, ds.rotation_3d_x, ds.rotation_3d_y, ds.rotation_3d_z, ds.flip_2d_perspective, ds.perspective_flip_theta, ds.perspective_flip_phi, ds.perspective_flip_gamma, ds.perspective_flip_fv, ds.noise_schedule, ds.strength_schedule, ds.contrast_schedule, ds.cfg_scale_schedule, ds.fov_schedule, ds.near_schedule, ds.far_schedule, ds.seed_schedule, ds.kernel_schedule, ds.sigma_schedule, ds.amount_schedule, ds.threshold_schedule, ds.histogram_matching, ds.color_coherence, ds.diffusion_cadence, ds.use_depth_warping, ds.midas_weight, ds.near_plane, ds.far_plane, ds.fov, ds.padding_mode, ds.sampling_mode, ds.save_depth_maps, ds.video_init_path, ds.extract_nth_frame, ds.overwrite_extracted_frames, ds.use_mask_video, ds.video_mask_path, ds.interpolate_key_frames, ds.interpolate_x_frames, ds.resume_from_timestring, ds.resume_timestring, ds.prompts, ds.animation_prompts, ds.W, ds.H, ds.restore_faces, ds.tiling, ds.enable_hr, ds.firstphase_width, ds.firstphase_height, ds.seed, ds.sampler, ds.seed_enable_extras, ds.subseed, ds.subseed_strength, ds.seed_resize_from_w, ds.seed_resize_from_h, ds.steps, ds.ddim_eta, ds.n_batch, ds.make_grid, ds.grid_rows, ds.save_settings, ds.save_samples, ds.display_samples, ds.save_sample_per_step, ds.show_sample_per_step, ds.override_these_with_webui, ds.batch_name, ds.filename_format, ds.seed_behavior, ds.use_init, ds.from_img2img_instead_of_link, ds.strength_0_no_init, ds.strength, ds.init_image, ds.use_mask, ds.use_alpha_as_mask, ds.invert_mask, ds.overlay_mask, ds.mask_file, ds.mask_contrast_adjust, ds.mask_brightness_adjust, ds.mask_overlay_blur, ds.fill, ds.full_res_mask, ds.full_res_mask_padding, ds.reroll_blank_frames, ds.smart_border_fill_mode, ds.skip_video_for_run_all, ds.fps, ds.output_format, ds.ffmpeg_location, ds.add_soundtrack, ds.soundtrack_path, ds.use_manual_settings, ds.render_steps, ds.max_video_frames, ds.path_name_modifier, ds.image_path, ds.mp4_path, ds.parseq_manifest, ds.parseq_use_deltas, ds.i1, ds.i2, ds.i3, ds.i4, ds.i5, ds.i6, ds.i7, ds.i8, ds.i9, ds.i10, ds.i11, ds.i12, ds.i13, ds.i14, ds.i15, ds.i16, ds.i17, ds.i18, ds.i19, ds.i20, ds.i21, ds.i22, ds.i23, ds.i24, ds.i25, ds.i26, ds.i27, ds.i28, ds.i29, ds.i30, ds.i31, ds.i32, ds.i33, ds.i34, ds.i35, ds.i36, ds.i37, ds.i38]
 
         submit.click(
                     fn=wrap_gradio_gpu_call(run_deforum),
@@ -375,7 +375,7 @@ def close_vid():
                     ],
                 )
         
-        settings_component_list = [ds.override_settings_with_file, ds.custom_settings_file, ds.animation_mode, ds.max_frames, ds.border, ds.angle, ds.zoom, ds.translation_x, ds.translation_y, ds.translation_z, ds.rotation_3d_x, ds.rotation_3d_y, ds.rotation_3d_z, ds.flip_2d_perspective, ds.perspective_flip_theta, ds.perspective_flip_phi, ds.perspective_flip_gamma, ds.perspective_flip_fv, ds.noise_schedule, ds.strength_schedule, ds.contrast_schedule, ds.cfg_scale_schedule, ds.fov_schedule, ds.near_schedule, ds.far_schedule, ds.seed_schedule, ds.kernel_schedule, ds.sigma_schedule, ds.amount_schedule, ds.threshold_schedule, ds.histogram_matching, ds.color_coherence, ds.diffusion_cadence, ds.use_depth_warping, ds.midas_weight, ds.near_plane, ds.far_plane, ds.fov, ds.padding_mode, ds.sampling_mode, ds.save_depth_maps, ds.video_init_path, ds.extract_nth_frame, ds.overwrite_extracted_frames, ds.use_mask_video, ds.video_mask_path, ds.interpolate_key_frames, ds.interpolate_x_frames, ds.resume_from_timestring, ds.resume_timestring, ds.prompts, ds.animation_prompts, ds.W, ds.H, ds.restore_faces, ds.tiling, ds.enable_hr, ds.firstphase_width, ds.firstphase_height, ds.seed, ds.sampler, ds.seed_enable_extras, ds.subseed, ds.subseed_strength, ds.seed_resize_from_w, ds.seed_resize_from_h, ds.steps, ds.ddim_eta, ds.n_batch, ds.make_grid, ds.grid_rows, ds.save_settings, ds.save_samples, ds.display_samples, ds.save_sample_per_step, ds.show_sample_per_step, ds.override_these_with_webui, ds.batch_name, ds.filename_format, ds.seed_behavior, ds.use_init, ds.from_img2img_instead_of_link, ds.strength_0_no_init, ds.strength, ds.init_image, ds.use_mask, ds.use_alpha_as_mask, ds.invert_mask, ds.overlay_mask, ds.mask_file, ds.mask_contrast_adjust, ds.mask_brightness_adjust, ds.mask_overlay_blur, ds.fill, ds.full_res_mask, ds.full_res_mask_padding, ds.reroll_blank_frames, ds.parseq_manifest, ds.parseq_use_deltas]
+        settings_component_list = [ds.override_settings_with_file, ds.custom_settings_file, ds.animation_mode, ds.max_frames, ds.border, ds.angle, ds.zoom, ds.translation_x, ds.translation_y, ds.translation_z, ds.rotation_3d_x, ds.rotation_3d_y, ds.rotation_3d_z, ds.flip_2d_perspective, ds.perspective_flip_theta, ds.perspective_flip_phi, ds.perspective_flip_gamma, ds.perspective_flip_fv, ds.noise_schedule, ds.strength_schedule, ds.contrast_schedule, ds.cfg_scale_schedule, ds.fov_schedule, ds.near_schedule, ds.far_schedule, ds.seed_schedule, ds.kernel_schedule, ds.sigma_schedule, ds.amount_schedule, ds.threshold_schedule, ds.histogram_matching, ds.color_coherence, ds.diffusion_cadence, ds.use_depth_warping, ds.midas_weight, ds.near_plane, ds.far_plane, ds.fov, ds.padding_mode, ds.sampling_mode, ds.save_depth_maps, ds.video_init_path, ds.extract_nth_frame, ds.overwrite_extracted_frames, ds.use_mask_video, ds.video_mask_path, ds.interpolate_key_frames, ds.interpolate_x_frames, ds.resume_from_timestring, ds.resume_timestring, ds.prompts, ds.animation_prompts, ds.W, ds.H, ds.restore_faces, ds.tiling, ds.enable_hr, ds.firstphase_width, ds.firstphase_height, ds.seed, ds.sampler, ds.seed_enable_extras, ds.subseed, ds.subseed_strength, ds.seed_resize_from_w, ds.seed_resize_from_h, ds.steps, ds.ddim_eta, ds.n_batch, ds.make_grid, ds.grid_rows, ds.save_settings, ds.save_samples, ds.display_samples, ds.save_sample_per_step, ds.show_sample_per_step, ds.override_these_with_webui, ds.batch_name, ds.filename_format, ds.seed_behavior, ds.use_init, ds.from_img2img_instead_of_link, ds.strength_0_no_init, ds.strength, ds.init_image, ds.use_mask, ds.use_alpha_as_mask, ds.invert_mask, ds.overlay_mask, ds.mask_file, ds.mask_contrast_adjust, ds.mask_brightness_adjust, ds.mask_overlay_blur, ds.fill, ds.full_res_mask, ds.full_res_mask_padding, ds.reroll_blank_frames, ds.smart_border_fill_mode, ds.parseq_manifest, ds.parseq_use_deltas]
         video_settings_component_list = [ds.skip_video_for_run_all, ds.fps, ds.output_format, ds.ffmpeg_location, ds.add_soundtrack, ds.soundtrack_path, ds.use_manual_settings, ds.render_steps, ds.max_video_frames, ds.path_name_modifier, ds.image_path, ds.mp4_path]
         stuff = gr.HTML("") # wrap gradio call garbage
         stuff.visible = False
diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py
index 816bcbe45..ed2826cc9 100644
--- a/scripts/deforum_helpers/args.py
+++ b/scripts/deforum_helpers/args.py
@@ -170,6 +170,7 @@ def DeforumArgs():
     full_res_mask = True
     full_res_mask_padding = 4
     reroll_blank_frames = 'reroll' # reroll, interrupt, or ignore
+    smart_border_fill_mode = 2
 
     n_samples = 1 # doesnt do anything
     precision = 'autocast' 
@@ -395,6 +396,9 @@ def show_vid():
         with gr.Row():
             # what to do with blank frames (they may result from glitches or the NSFW filter being turned on): reroll with +1 seed, interrupt the animation generation, or do nothing
             reroll_blank_frames = gr.Dropdown(label="reroll_blank_frames", choices=['reroll', 'interrupt', 'ignore'], value=d.reroll_blank_frames, type="value", elem_id="reroll_blank_frames", interactive=True)
+        with gr.Row():
+            choice = mask_fill_choices[d.smart_border_fill_mode]
+            smart_border_fill_mode = gr.Radio(label='smart_border_fill_mode', choices=mask_fill_choices, value=choice, type="index")
         with gr.Row():
             histogram_matching = gr.Checkbox(label="Force all frames to match initial frame's colors. Overrides a1111 settings. NOT RECOMMENDED, enable only for backwards compatibility.", value=da.histogram_matching, interactive=True)
         with gr.Row():            
@@ -536,12 +540,12 @@ def show_vid():
     
 def setup_deforum_setting_ui(self, is_img2img, is_extension = True):
     ds = SimpleNamespace(**setup_deforum_setting_dictionary(self, is_img2img, is_extension))
-    return [ds.btn, ds.override_settings_with_file, ds.custom_settings_file, ds.animation_mode, ds.max_frames, ds.border, ds.angle, ds.zoom, ds.translation_x, ds.translation_y, ds.translation_z, ds.rotation_3d_x, ds.rotation_3d_y, ds.rotation_3d_z, ds.flip_2d_perspective, ds.perspective_flip_theta, ds.perspective_flip_phi, ds.perspective_flip_gamma, ds.perspective_flip_fv, ds.noise_schedule, ds.strength_schedule, ds.contrast_schedule, ds.cfg_scale_schedule, ds.fov_schedule, ds.near_schedule, ds.far_schedule, ds.seed_schedule, ds.kernel_schedule, ds.sigma_schedule, ds.amount_schedule, ds.threshold_schedule, ds.histogram_matching, ds.color_coherence, ds.diffusion_cadence, ds.use_depth_warping, ds.midas_weight, ds.near_plane, ds.far_plane, ds.fov, ds.padding_mode, ds.sampling_mode, ds.save_depth_maps, ds.video_init_path, ds.extract_nth_frame, ds.overwrite_extracted_frames, ds.use_mask_video, ds.video_mask_path, ds.interpolate_key_frames, ds.interpolate_x_frames, ds.resume_from_timestring, ds.resume_timestring, ds.prompts, ds.animation_prompts, ds.W, ds.H, ds.restore_faces, ds.tiling, ds.enable_hr, ds.firstphase_width, ds.firstphase_height, ds.seed, ds.sampler, ds.seed_enable_extras, ds.subseed, ds.subseed_strength, ds.seed_resize_from_w, ds.seed_resize_from_h, ds.steps, ds.ddim_eta, ds.n_batch, ds.make_grid, ds.grid_rows, ds.save_settings, ds.save_samples, ds.display_samples, ds.save_sample_per_step, ds.show_sample_per_step, ds.override_these_with_webui, ds.batch_name, ds.filename_format, ds.seed_behavior, ds.use_init, ds.from_img2img_instead_of_link, ds.strength_0_no_init, ds.strength, ds.init_image, ds.use_mask, ds.use_alpha_as_mask, ds.invert_mask, ds.overlay_mask, ds.mask_file, ds.mask_contrast_adjust, ds.mask_brightness_adjust, ds.mask_overlay_blur, ds.fill, ds.full_res_mask, ds.full_res_mask_padding, ds.reroll_blank_frames, ds.skip_video_for_run_all, ds.fps, ds.output_format, ds.ffmpeg_location, ds.add_soundtrack, ds.soundtrack_path, ds.use_manual_settings, ds.render_steps, ds.max_video_frames, ds.path_name_modifier, ds.image_path, ds.mp4_path, ds.parseq_manifest, ds.parseq_use_deltas, ds.i1, ds.i2, ds.i3, ds.i4, ds.i5, ds.i6, ds.i7, ds.i8, ds.i9, ds.i10, ds.i11, ds.i12, ds.i13, ds.i14, ds.i15, ds.i16, ds.i17, ds.i18, ds.i19, ds.i20, ds.i21, ds.i22, ds.i23, ds.i24, ds.i25, ds.i26, ds.i27, ds.i28, ds.i29, ds.i30, ds.i31, ds.i32, ds.i33, ds.i34, ds.i35, ds.i36, ds.i37, ds.i38]
+    return [ds.btn, ds.override_settings_with_file, ds.custom_settings_file, ds.animation_mode, ds.max_frames, ds.border, ds.angle, ds.zoom, ds.translation_x, ds.translation_y, ds.translation_z, ds.rotation_3d_x, ds.rotation_3d_y, ds.rotation_3d_z, ds.flip_2d_perspective, ds.perspective_flip_theta, ds.perspective_flip_phi, ds.perspective_flip_gamma, ds.perspective_flip_fv, ds.noise_schedule, ds.strength_schedule, ds.contrast_schedule, ds.cfg_scale_schedule, ds.fov_schedule, ds.near_schedule, ds.far_schedule, ds.seed_schedule, ds.kernel_schedule, ds.sigma_schedule, ds.amount_schedule, ds.threshold_schedule, ds.histogram_matching, ds.color_coherence, ds.diffusion_cadence, ds.use_depth_warping, ds.midas_weight, ds.near_plane, ds.far_plane, ds.fov, ds.padding_mode, ds.sampling_mode, ds.save_depth_maps, ds.video_init_path, ds.extract_nth_frame, ds.overwrite_extracted_frames, ds.use_mask_video, ds.video_mask_path, ds.interpolate_key_frames, ds.interpolate_x_frames, ds.resume_from_timestring, ds.resume_timestring, ds.prompts, ds.animation_prompts, ds.W, ds.H, ds.restore_faces, ds.tiling, ds.enable_hr, ds.firstphase_width, ds.firstphase_height, ds.seed, ds.sampler, ds.seed_enable_extras, ds.subseed, ds.subseed_strength, ds.seed_resize_from_w, ds.seed_resize_from_h, ds.steps, ds.ddim_eta, ds.n_batch, ds.make_grid, ds.grid_rows, ds.save_settings, ds.save_samples, ds.display_samples, ds.save_sample_per_step, ds.show_sample_per_step, ds.override_these_with_webui, ds.batch_name, ds.filename_format, ds.seed_behavior, ds.use_init, ds.from_img2img_instead_of_link, ds.strength_0_no_init, ds.strength, ds.init_image, ds.use_mask, ds.use_alpha_as_mask, ds.invert_mask, ds.overlay_mask, ds.mask_file, ds.mask_contrast_adjust, ds.mask_brightness_adjust, ds.mask_overlay_blur, ds.fill, ds.full_res_mask, ds.full_res_mask_padding, ds.reroll_blank_frames, ds.smart_border_fill_mode, ds.skip_video_for_run_all, ds.fps, ds.output_format, ds.ffmpeg_location, ds.add_soundtrack, ds.soundtrack_path, ds.use_manual_settings, ds.render_steps, ds.max_video_frames, ds.path_name_modifier, ds.image_path, ds.mp4_path, ds.parseq_manifest, ds.parseq_use_deltas, ds.i1, ds.i2, ds.i3, ds.i4, ds.i5, ds.i6, ds.i7, ds.i8, ds.i9, ds.i10, ds.i11, ds.i12, ds.i13, ds.i14, ds.i15, ds.i16, ds.i17, ds.i18, ds.i19, ds.i20, ds.i21, ds.i22, ds.i23, ds.i24, ds.i25, ds.i26, ds.i27, ds.i28, ds.i29, ds.i30, ds.i31, ds.i32, ds.i33, ds.i34, ds.i35, ds.i36, ds.i37, ds.i38]
 
 def pack_anim_args(animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring):
     return locals()
 
-def pack_args(W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask,  mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames):
+def pack_args(W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask,  mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode):
     precision = 'autocast' 
     scale = 7
     C = 4
@@ -559,9 +563,9 @@ def pack_video_args(skip_video_for_run_all, fps, output_format, ffmpeg_location,
 def pack_parseq_args(parseq_manifest, parseq_use_deltas):
     return locals()
 
-def process_args(self, p, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest, parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38):
+def process_args(self, p, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode, skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path, parseq_manifest, parseq_use_deltas, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21, i22, i23, i24, i25, i26, i27, i28, i29, i30, i31, i32, i33, i34, i35, i36, i37, i38):
 
-    args_dict = pack_args(W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames)
+    args_dict = pack_args(W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode)
     anim_args_dict = pack_anim_args(animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring)
     video_args_dict = pack_video_args(skip_video_for_run_all, fps, output_format, ffmpeg_location, add_soundtrack, soundtrack_path, use_manual_settings, render_steps, max_video_frames, path_name_modifier, image_path, mp4_path)
     parseq_args_dict = pack_parseq_args(parseq_manifest, parseq_use_deltas)
diff --git a/scripts/deforum_helpers/settings.py b/scripts/deforum_helpers/settings.py
index 4012ea03f..aa63fc368 100644
--- a/scripts/deforum_helpers/settings.py
+++ b/scripts/deforum_helpers/settings.py
@@ -34,9 +34,9 @@ def load_args(args_dict,anim_args_dict, parseq_args_dict, custom_settings_file,
 import gradio as gr
  
 # In gradio gui settings save/load
-def save_settings(settings_path, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, parseq_manifest, parseq_use_deltas):
+def save_settings(settings_path, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode, parseq_manifest, parseq_use_deltas):
     from deforum_helpers.args import pack_args, pack_anim_args, pack_parseq_args
-    args_dict = pack_args(W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames)
+    args_dict = pack_args(W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode)
     anim_args_dict = pack_anim_args(animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring)
     parseq_dict = pack_parseq_args(parseq_manifest, parseq_use_deltas)    
     args_dict["prompts"] = json.loads(animation_prompts)
@@ -53,14 +53,14 @@ def save_video_settings(video_settings_path, skip_video_for_run_all, fps, output
         f.write(json.dumps(video_args_dict, ensure_ascii=False, indent=4))
     return [""]
 
-def load_settings(settings_path, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, parseq_manifest, parseq_use_deltas):
+def load_settings(settings_path, override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode, parseq_manifest, parseq_use_deltas):
     print(f"reading custom settings from {settings_path}")
     data = locals()
     data.pop("settings_path")
     jdata = {}
     if not os.path.isfile(settings_path):
         print('The custom settings file does not exist. The values will be unchanged.')
-        return [override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, parseq_manifest, parseq_use_deltas, ""]
+        return [override_settings_with_file, custom_settings_file, animation_mode, max_frames, border, angle, zoom, translation_x, translation_y, translation_z, rotation_3d_x, rotation_3d_y, rotation_3d_z, flip_2d_perspective, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv, noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, fov_schedule, near_schedule, far_schedule, seed_schedule, kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule, histogram_matching, color_coherence, diffusion_cadence, use_depth_warping, midas_weight, near_plane, far_plane, fov, padding_mode, sampling_mode, save_depth_maps, video_init_path, extract_nth_frame, overwrite_extracted_frames, use_mask_video, video_mask_path, interpolate_key_frames, interpolate_x_frames, resume_from_timestring, resume_timestring, prompts, animation_prompts, W, H, restore_faces, tiling, enable_hr, firstphase_width, firstphase_height, seed, sampler, seed_enable_extras, subseed, subseed_strength, seed_resize_from_w, seed_resize_from_h, steps, ddim_eta, n_batch, make_grid, grid_rows, save_settings, save_samples, display_samples, save_sample_per_step, show_sample_per_step, override_these_with_webui, batch_name, filename_format, seed_behavior, use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image, use_mask, use_alpha_as_mask, invert_mask, overlay_mask, mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur, fill, full_res_mask, full_res_mask_padding, reroll_blank_frames, smart_border_fill_mode, parseq_manifest, parseq_use_deltas, ""]
     else:
         with open(settings_path, "r") as f:
             jdata = json.loads(f.read())
@@ -90,6 +90,18 @@ def load_settings(settings_path, override_settings_with_file, custom_settings_fi
                 logging.debug(f"Fill not found in load file, using default value: {fill_default}")
                 ret.append(mask_fill_choices[fill_default])
         
+        elif key == 'smart_border_fill_mode':
+            if key in jdata:
+                smart_border_fill_mode_val = jdata[key]
+                if type(smart_border_fill_mode_val) == int:                    
+                    ret.append(mask_fill_choices[smart_border_fill_mode_val])
+                else:
+                    ret.append(smart_border_fill_mode_val)
+            else:
+                smart_border_fill_mode_default = DeforumArgs()['smart_border_fill_mode']
+                logging.debug(f"Smart border fill mode not found in load file, using default value: {smart_border_fill_mode_default}")
+                ret.append(mask_fill_choices[smart_border_fill_mode_default])
+        
         elif key == 'reroll_blank_frames':
             if key in jdata:
                 reroll_blank_frames_val = jdata[key]

From 7d0ee334cfe60e4b095a50e61c8463e6bbfc1616 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 22:28:32 +0300
Subject: [PATCH 05/19] zeros mask smallness check

---
 scripts/deforum_helpers/generate.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index 77d3fe387..960381ef0 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -22,7 +22,7 @@
 #Webui
 import cv2
 from .animation import sample_from_cv2, sample_to_cv2
-from modules import processing
+from modules import processing, masking
 from modules.shared import opts, sd_model
 from modules.processing import process_images, StableDiffusionProcessingTxt2Img
 import logging
@@ -179,19 +179,29 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
                                     args.mask_contrast_adjust, 
                                     args.mask_brightness_adjust, 
                                     invert_mask=False)
+            
+            # HACK: this is a hacky check to make the mask work with the new inpainting code
+            crop_region = masking.get_crop_region(np.array(mask_image), args.full_res_mask_padding)
+            crop_region = masking.expand_crop_region(crop_region, args.W, args.H, mask_image.width, mask_image.height)
+            x1, y1, x2, y2 = crop_region
 
-            p.inpainting_fill = args.fill # need to come up with better name. 
-            p.inpaint_full_res= args.full_res_mask 
-            p.inpaint_full_res_padding = args.full_res_mask_padding 
+            too_small = (x2 - x1) < 1 or (y2 - y1) < 1
+
+            if not too_small:
+                p.inpainting_fill = args.fill # need to come up with better name. 
+                p.inpaint_full_res= args.full_res_mask 
+                p.inpaint_full_res_padding = args.full_res_mask_padding 
 
-            p.init_images = [init_image]
-            p.image_mask = mask
+                p.init_images = [init_image]
+                p.image_mask = mask
 
-            processed = processing.process_images(p)
+                processed = processing.process_images(p)
 
-            init_image = processed.images[0].convert('RGB') 
+                init_image = processed.images[0].convert('RGB') 
             p.image_mask = None
+            p.inpainting_fill = 0
             mask_image = None
+            mask = None
             processed = None
     elif args.use_init and args.init_image != None and args.init_image != '':
         init_image, mask_image = load_img(args.init_image, 

From 6e29361dfd75d84ae416ddb117fb5dbe74f0fb79 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 22:36:26 +0300
Subject: [PATCH 06/19] correct tqdm when inpainting pass is skipped

---
 scripts/deforum_helpers/generate.py | 12 +++++++++++-
 scripts/deforum_helpers/settings.py |  7 +++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index 960381ef0..6e2905e80 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -3,6 +3,7 @@
 from PIL import Image, ImageOps
 import requests
 import numpy as np
+from math import ceil
 import torchvision.transforms.functional as TF
 from pytorch_lightning import seed_everything
 import os
@@ -23,6 +24,7 @@
 import cv2
 from .animation import sample_from_cv2, sample_to_cv2
 from modules import processing, masking
+from modules import shared
 from modules.shared import opts, sd_model
 from modules.processing import process_images, StableDiffusionProcessingTxt2Img
 import logging
@@ -195,9 +197,17 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
                 p.init_images = [init_image]
                 p.image_mask = mask
 
+                print("Smart mode: inpainting border")
+
                 processed = processing.process_images(p)
 
-                init_image = processed.images[0].convert('RGB') 
+                init_image = processed.images[0].convert('RGB')
+            else:
+                # fix tqdm total steps if we don't have to conduct a second pass
+                tqdm_instance = shared.total_tqdm
+                current_total = tqdm_instance.getTotal()
+                if current_total != -1:
+                    tqdm_instance.updateTotal(current_total - int(ceil(args.steps * (1-args.strength))))
             p.image_mask = None
             p.inpainting_fill = 0
             mask_image = None
diff --git a/scripts/deforum_helpers/settings.py b/scripts/deforum_helpers/settings.py
index aa63fc368..d972181ed 100644
--- a/scripts/deforum_helpers/settings.py
+++ b/scripts/deforum_helpers/settings.py
@@ -230,3 +230,10 @@ def clear(self):
         if self._tqdm is not None:
             self._tqdm.close()
             self._tqdm = None
+    
+    def getTotal(self):
+        if not opts.multiple_tqdm or cmd_opts.disable_console_progressbars:
+            return -1
+        if self._tqdm is None:
+            self.reset()
+        return self._tqdm.total

From dc4638b25a0dba26ba0ef0fea22873c842d07e15 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 22:40:04 +0300
Subject: [PATCH 07/19]  reset model after zeros inpaint

---
 scripts/deforum_helpers/generate.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index 6e2905e80..2adb681b7 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -213,6 +213,7 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
             mask_image = None
             mask = None
             processed = None
+            p.sd_model=sd_model
     elif args.use_init and args.init_image != None and args.init_image != '':
         init_image, mask_image = load_img(args.init_image, 
                                           shape=(args.W, args.H),  

From 65f2dc5fc6af2ca325ac1f53b4ac8bea2d7e7b93 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 23:20:37 +0300
Subject: [PATCH 08/19] get warp masks from transform_image_3d directly

Co-Authored-By: Funofabot <117096230+Funofabot@users.noreply.github.com>
---
 scripts/deforum_helpers/animation.py | 24 ++++++++++++++++++++----
 scripts/deforum_helpers/args.py      |  1 +
 scripts/deforum_helpers/generate.py  | 12 ++++++++++++
 scripts/deforum_helpers/render.py    |  3 ++-
 4 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/scripts/deforum_helpers/animation.py b/scripts/deforum_helpers/animation.py
index ce559ad3a..9e736eb5e 100644
--- a/scripts/deforum_helpers/animation.py
+++ b/scripts/deforum_helpers/animation.py
@@ -210,6 +210,8 @@ def anim_frame_warp(prev, args, anim_args, keys, frame_idx, depth_model=None, de
     else:
         prev_img_cv2 = sample_to_cv2(prev)
 
+    warp_mask = None
+
     if anim_args.use_depth_warping:
         if depth is None and depth_model is not None:
             depth = depth_model.predict(prev_img_cv2, anim_args, half_precision)
@@ -219,9 +221,8 @@ def anim_frame_warp(prev, args, anim_args, keys, frame_idx, depth_model=None, de
     if anim_args.animation_mode == '2D':
         prev_img = anim_frame_warp_2d(prev_img_cv2, args, anim_args, keys, frame_idx)
     else: # '3D'
-        prev_img = anim_frame_warp_3d(device, prev_img_cv2, depth, anim_args, keys, frame_idx)
-                
-    return prev_img, depth
+        prev_img, warp_mask = anim_frame_warp_3d(device, prev_img_cv2, depth, anim_args, keys, frame_idx)        
+    return prev_img, depth, warp_mask
 
 def anim_frame_warp_2d(prev_img_cv2, args, anim_args, keys, frame_idx):
     angle = keys.angle_series[frame_idx]
@@ -330,7 +331,22 @@ def transform_image_3d(device, prev_img_cv2, depth_tensor, rot_mat, translate, a
         new_image.squeeze().clamp(0,255), 
         'c h w -> h w c'
     ).cpu().numpy().astype(prev_img_cv2.dtype)
-    return result
+
+    warp_mask = torch.ones_like(image_tensor) * 255
+    warp_mask_image = torch.nn.functional.grid_sample(
+        warp_mask.add(1/512 - 0.0001).unsqueeze(0), 
+        offset_coords_2d, 
+        mode="nearest", 
+        padding_mode="zeros", 
+        align_corners=False
+    )
+
+    result_mask = rearrange(
+        warp_mask_image.squeeze().clamp(0,255), 
+        'c h w -> h w c'
+    ).cpu().numpy().astype(prev_img_cv2.dtype)
+
+    return result, result_mask
 
 class DeformAnimKeys():
     def __init__(self, anim_args):
diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py
index ed2826cc9..1250abf11 100644
--- a/scripts/deforum_helpers/args.py
+++ b/scripts/deforum_helpers/args.py
@@ -15,6 +15,7 @@ def Root():
     outpath_samples = ""
     animation_prompts = None
     color_corrections = None
+    warp_mask = None
     return locals()
 
 def DeforumAnimArgs():
diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index 2adb681b7..331dde38f 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -176,6 +176,18 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
                     else:
                         mask_image.putpixel((x,y), 0)
             
+            # blend the two masks
+            if root.warp_mask is not None:
+                # TODO: I guess there is some built-in function for this
+                warp_mask_image = Image.fromarray(cv2.cvtColor(sample_to_cv2(root.warp_mask), cv2.COLOR_BGR2RGB)).convert('L')
+                for x in range(mask_image.width):
+                    for y in range(mask_image.height):
+                        if mask_image.getpixel((x,y)) > 0 or warp_mask_image.getpixel((x,y)) > 0:
+                            mask_image.putpixel((x,y), 255)
+                        else:
+                            mask_image.putpixel((x,y), 0)
+                root.warp_mask = None
+            
             mask = prepare_mask(mask_image, 
                                     (args.W, args.H), 
                                     args.mask_contrast_adjust, 
diff --git a/scripts/deforum_helpers/render.py b/scripts/deforum_helpers/render.py
index 2fb12adbe..a2a63905e 100644
--- a/scripts/deforum_helpers/render.py
+++ b/scripts/deforum_helpers/render.py
@@ -188,7 +188,7 @@ def render_animation(args, anim_args, parseq_args, animation_prompts, root):
 
         # apply transforms to previous frame
         if prev_sample is not None:
-            prev_img, depth = anim_frame_warp(prev_sample, args, anim_args, keys, frame_idx, depth_model, depth=None, device=root.device, half_precision=root.half_precision)
+            prev_img, depth, warp_mask = anim_frame_warp(prev_sample, args, anim_args, keys, frame_idx, depth_model, depth=None, device=root.device, half_precision=root.half_precision)
 
             # apply color matching
             if anim_args.color_coherence != 'None':
@@ -211,6 +211,7 @@ def render_animation(args, anim_args, parseq_args, animation_prompts, root):
             else:
                 args.init_sample = noised_sample.to(root.device)
             args.strength = max(0.0, min(1.0, strength))
+            root.warp_mask = warp_mask
         
         args.scale = scale
 

From 50f8d6336d66155e0d9e4bcdb578310e0bae913d Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 23:35:56 +0300
Subject: [PATCH 09/19] move smart_border_fill_mode near mask_fill

---
 scripts/deforum_helpers/args.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py
index 1250abf11..f435047d7 100644
--- a/scripts/deforum_helpers/args.py
+++ b/scripts/deforum_helpers/args.py
@@ -397,9 +397,6 @@ def show_vid():
         with gr.Row():
             # what to do with blank frames (they may result from glitches or the NSFW filter being turned on): reroll with +1 seed, interrupt the animation generation, or do nothing
             reroll_blank_frames = gr.Dropdown(label="reroll_blank_frames", choices=['reroll', 'interrupt', 'ignore'], value=d.reroll_blank_frames, type="value", elem_id="reroll_blank_frames", interactive=True)
-        with gr.Row():
-            choice = mask_fill_choices[d.smart_border_fill_mode]
-            smart_border_fill_mode = gr.Radio(label='smart_border_fill_mode', choices=mask_fill_choices, value=choice, type="index")
         with gr.Row():
             histogram_matching = gr.Checkbox(label="Force all frames to match initial frame's colors. Overrides a1111 settings. NOT RECOMMENDED, enable only for backwards compatibility.", value=da.histogram_matching, interactive=True)
         with gr.Row():            
@@ -488,6 +485,9 @@ def show_vid():
         with gr.Row():
             choice = mask_fill_choices[d.fill]
             fill = gr.Radio(label='mask_fill', choices=mask_fill_choices, value=choice, type="index")
+        with gr.Row():
+            choice = mask_fill_choices[d.smart_border_fill_mode]
+            smart_border_fill_mode = gr.Radio(label='smart_border_fill_mode', choices=mask_fill_choices, value=choice, type="index")
         with gr.Row():
             #fill = gr.Slider(minimum=0, maximum=3, step=1, label="mask_fill_type", value=d.fill, interactive=True)
             full_res_mask = gr.Checkbox(label="full_res_mask", value=d.full_res_mask, interactive=True)

From fb0bd4a1787d2f1cbd365963b19e2424c7fb5179 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Tue, 27 Dec 2022 23:42:30 +0300
Subject: [PATCH 10/19] fixup

---
 scripts/deforum_helpers/generate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index 331dde38f..25bb14789 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -179,7 +179,7 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
             # blend the two masks
             if root.warp_mask is not None:
                 # TODO: I guess there is some built-in function for this
-                warp_mask_image = Image.fromarray(cv2.cvtColor(sample_to_cv2(root.warp_mask), cv2.COLOR_BGR2RGB)).convert('L')
+                warp_mask_image = Image.fromarray(root.warp_mask).convert('L')
                 for x in range(mask_image.width):
                     for y in range(mask_image.height):
                         if mask_image.getpixel((x,y)) > 0 or warp_mask_image.getpixel((x,y)) > 0:

From c3d8f2922224e2fd91bbb1e61c1adb28282b60e4 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Wed, 28 Dec 2022 00:50:01 +0300
Subject: [PATCH 11/19] move pipeline reset to a dedicated function

---
 scripts/deforum_helpers/generate.py | 59 +++++++++++++++--------------
 1 file changed, 31 insertions(+), 28 deletions(-)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index 25bb14789..4963fba6f 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -96,21 +96,8 @@ def prepare_mask(mask_input, mask_shape, mask_brightness_adjust=1.0, mask_contra
         mask = PIL.ImageOps.invert(mask)
     
     return mask
-    
-def generate(args, anim_args, root, frame = 0, return_sample=False):
-    import re
-    assert args.prompt is not None
-    
-    # Evaluate prompt math!
-    
-    math_parser = re.compile("""
-            (?P<weight>(
-            `[\S\s]*?`# a math function wrapped in `-characters
-            ))
-            """, re.VERBOSE)
-    
-    parsed_prompt = re.sub(math_parser, lambda m: str(parse_weight(m, frame)), args.prompt)
-    
+
+def reset_pipeline(root, args, prompt, negative_prompt):
     # Setup the pipeline
     p = root.p
     
@@ -139,15 +126,34 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
     p.outpath_samples = root.outpath_samples
     p.outpath_grids = root.outpath_samples
     
+    p.prompt = prompt
+    p.negative_prompt = negative_prompt
+
+def generate(args, anim_args, root, frame = 0, return_sample=False):
+    import re
+    assert args.prompt is not None
+    
+    # Evaluate prompt math!
+    
+    math_parser = re.compile("""
+            (?P<weight>(
+            `[\S\s]*?`# a math function wrapped in `-characters
+            ))
+            """, re.VERBOSE)
+    
+    parsed_prompt = re.sub(math_parser, lambda m: str(parse_weight(m, frame)), args.prompt)
+
     prompt_split = parsed_prompt.split("--neg")
     if len(prompt_split) > 1:
-        p.prompt, p.negative_prompt = parsed_prompt.split("--neg") #TODO: add --neg to vanilla Deforum for compat
-        print(f'Positive prompt:{p.prompt}')
-        print(f'Negative prompt:{p.negative_prompt}')
+        prompt, negative_prompt = parsed_prompt.split("--neg") #TODO: add --neg to vanilla Deforum for compat
+        print(f'Positive prompt:{prompt}')
+        print(f'Negative prompt:{negative_prompt}')
     else:
-        p.prompt = prompt_split[0]
-        print(f'Positive prompt:{p.prompt}')
-        p.negative_prompt = ""
+        prompt = prompt_split[0]
+        print(f'Positive prompt:{prompt}')
+        negative_prompt = ""
+    
+    p = reset_pipeline(root, args, prompt, negative_prompt)
     
     if not args.use_init and args.strength > 0 and args.strength_0_no_init:
         print("\nNo init image, but strength > 0. Strength has been auto set to 0, since use_init is False.")
@@ -212,20 +218,17 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
                 print("Smart mode: inpainting border")
 
                 processed = processing.process_images(p)
-
                 init_image = processed.images[0].convert('RGB')
+                p = reset_pipeline(root, args, prompt, negative_prompt)
+                processed = None
+                mask = None
+                mask_image = None
             else:
                 # fix tqdm total steps if we don't have to conduct a second pass
                 tqdm_instance = shared.total_tqdm
                 current_total = tqdm_instance.getTotal()
                 if current_total != -1:
                     tqdm_instance.updateTotal(current_total - int(ceil(args.steps * (1-args.strength))))
-            p.image_mask = None
-            p.inpainting_fill = 0
-            mask_image = None
-            mask = None
-            processed = None
-            p.sd_model=sd_model
     elif args.use_init and args.init_image != None and args.init_image != '':
         init_image, mask_image = load_img(args.init_image, 
                                           shape=(args.W, args.H),  

From fd8ebf556594ff0af81bc3ff1967eb266dbee7d9 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Wed, 28 Dec 2022 01:00:49 +0300
Subject: [PATCH 12/19] fixup missed return

---
 scripts/deforum_helpers/generate.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index 4963fba6f..af6969769 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -128,6 +128,7 @@ def reset_pipeline(root, args, prompt, negative_prompt):
     
     p.prompt = prompt
     p.negative_prompt = negative_prompt
+    return p
 
 def generate(args, anim_args, root, frame = 0, return_sample=False):
     import re

From d473dafc6ce47a3fa519dbb41b743ab33189b9b5 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Wed, 28 Dec 2022 01:17:44 +0300
Subject: [PATCH 13/19] reset mask in any case

---
 scripts/deforum_helpers/generate.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index af6969769..c43dae066 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -221,15 +221,17 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
                 processed = processing.process_images(p)
                 init_image = processed.images[0].convert('RGB')
                 p = reset_pipeline(root, args, prompt, negative_prompt)
+                p.image_mask = None
                 processed = None
-                mask = None
-                mask_image = None
             else:
                 # fix tqdm total steps if we don't have to conduct a second pass
                 tqdm_instance = shared.total_tqdm
                 current_total = tqdm_instance.getTotal()
                 if current_total != -1:
                     tqdm_instance.updateTotal(current_total - int(ceil(args.steps * (1-args.strength))))
+            
+            mask = None
+            mask_image = None
     elif args.use_init and args.init_image != None and args.init_image != '':
         init_image, mask_image = load_img(args.init_image, 
                                           shape=(args.W, args.H),  

From 13670f207b195231fdc61a0e617fd8d116f5d786 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Wed, 28 Dec 2022 02:10:21 +0300
Subject: [PATCH 14/19] full pipeline reset!

finally fixed not diffusing

Co-Authored-By: Funofabot <117096230+Funofabot@users.noreply.github.com>
---
 scripts/deforum_helpers/generate.py | 120 ++++++++++++++++++++--------
 1 file changed, 86 insertions(+), 34 deletions(-)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index c43dae066..6c4ceb742 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -26,7 +26,7 @@
 from modules import processing, masking
 from modules import shared
 from modules.shared import opts, sd_model
-from modules.processing import process_images, StableDiffusionProcessingTxt2Img
+from modules.processing import process_images, StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img
 import logging
 
 #MASKARGSEXPANSION 
@@ -97,35 +97,65 @@ def prepare_mask(mask_input, mask_shape, mask_brightness_adjust=1.0, mask_contra
     
     return mask
 
-def reset_pipeline(root, args, prompt, negative_prompt):
+# Resets the pipeline object as recomended by kabachuha to simplify resets for additional passes
+def reset_pipeline(args, prompt, negative_prompt):
     # Setup the pipeline
-    p = root.p
-    
-    os.makedirs(args.outdir, exist_ok=True)
-    p.batch_size = args.n_samples
-    p.width = args.W
-    p.height = args.H
-    p.seed = args.seed
-    p.subseed=args.subseed
-    p.subseed_strength=args.subseed_strength
-    p.do_not_save_samples = not args.save_sample_per_step
-    p.do_not_save_grid = not args.make_grid
-    p.sd_model=sd_model
-    p.sampler_name = args.sampler
-    p.mask_blur = args.mask_overlay_blur
-    p.extra_generation_params["Mask blur"] = args.mask_overlay_blur
-    p.n_iter = 1
-    p.steps = args.steps
-    p.denoising_strength = 1 - args.strength
-    p.cfg_scale = args.scale
+    p = StableDiffusionProcessingImg2Img(
+        sd_model=sd_model,
+        outpath_samples = opts.outdir_samples or opts.outdir_img2img_samples,
+        outpath_grids = opts.outdir_grids or opts.outdir_img2img_grids,
+        #prompt=prompt, 
+        #negative_prompt=negative_prompt,
+        #styles=[prompt_style, prompt_style2], 
+        seed=args.seed,
+        subseed=args.subseed,
+        subseed_strength=args.subseed_strength,
+        seed_resize_from_h=args.seed_resize_from_h,
+        seed_resize_from_w=args.seed_resize_from_w,
+        #seed_enable_extras=args.seed_enable_extras,
+        sampler_name = args.sampler,
+        batch_size=args.n_batch,
+        n_iter=1,
+        cfg_scale=args.scale,
+        width=args.W,
+        height=args.H,
+        restore_faces=args.restore_faces,
+        tiling=args.tiling,
+        #init_images=[image], # Assigned during generation 
+        mask=None, # Assigned during generation 
+        mask_blur=args.mask_overlay_blur,
+        #resize_mode=resize_mode, #TODO There are several settings to this and it may 
+        #inpainting_fill=args.fill, # Assign during generation
+        #inpaint_full_res=args.full_res_mask, # Assign during generation
+        #inpaint_full_res_padding=inpaint_full_res_padding, # Assign during generation
+        #inpainting_mask_invert=inpainting_mask_invert, # Assign during generation
+        do_not_save_samples=not args.save_sample_per_step,
+        do_not_save_grid=not args.make_grid,
+    )
+    p.restore_faces = args.restore_faces
+    p.tiling = args.tiling
+    p.enable_hr = args.enable_hr
+    p.firstphase_width = args.firstphase_width
+    p.firstphase_height = args.firstphase_height
     p.seed_enable_extras = args.seed_enable_extras
+    p.subseed = args.subseed
+    p.subseed_strength = args.subseed_strength
+    p.seed_resize_from_w = args.seed_resize_from_w
+    p.seed_resize_from_h = args.seed_resize_from_h
+    p.fill = args.fill
+    p.ddim_eta = args.ddim_eta
 
-    # FIXME better color corrections as match histograms doesn't seem to be fully working
-    if root.color_corrections is not None:
-        p.color_corrections = root.color_corrections
-    p.outpath_samples = root.outpath_samples
-    p.outpath_grids = root.outpath_samples
+    # Below settings which have conditions or symbols that dont work in the p object constructor
+    p.extra_generation_params["Mask blur"] = args.mask_overlay_blur
+
+    p.steps = args.steps
+    if opts.img2img_fix_steps:
+        p.denoising_strength = 1 / (1 - args.strength + 1.0/args.steps) #see https://github.com/deforum-art/deforum-for-automatic1111-webui/issues/3
+    else:
+        p.denoising_strength = 1 - args.strength
     
+    # The prompts are precomputed as the math functions can be quite resource heavy
+    # and also not to log things twice
     p.prompt = prompt
     p.negative_prompt = negative_prompt
     return p
@@ -154,7 +184,7 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
         print(f'Positive prompt:{prompt}')
         negative_prompt = ""
     
-    p = reset_pipeline(root, args, prompt, negative_prompt)
+    p = reset_pipeline(args, prompt, negative_prompt)
     
     if not args.use_init and args.strength > 0 and args.strength_0_no_init:
         print("\nNo init image, but strength > 0. Strength has been auto set to 0, since use_init is False.")
@@ -175,10 +205,14 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
             # Inpaint changed parts of the image
             # that's, to say, zeros we got after the transformations
 
+            # Its important to note that the loop below is creating a mask for inpainting 0's
+            # This mask however can mask areas that were intended to be black
+            # Suggest a fix to send the inpainting mask as an argument,
+            # before the add_noise and contrast_adjust is applied
             mask_image = init_image.convert('L')
             for x in range(mask_image.width):
                 for y in range(mask_image.height):
-                    if mask_image.getpixel((x,y)) < 1:
+                    if mask_image.getpixel((x,y)) < 4:
                         mask_image.putpixel((x,y), 255)
                     else:
                         mask_image.putpixel((x,y), 0)
@@ -209,20 +243,38 @@ def generate(args, anim_args, root, frame = 0, return_sample=False):
             too_small = (x2 - x1) < 1 or (y2 - y1) < 1
 
             if not too_small:
-                p.inpainting_fill = args.fill # need to come up with better name. 
+                p.do_not_save_samples=True,         
+                p.inpainting_fill = args.smart_border_fill_mode 
                 p.inpaint_full_res= args.full_res_mask 
                 p.inpaint_full_res_padding = args.full_res_mask_padding 
-
                 p.init_images = [init_image]
-                p.image_mask = mask
+                p.image_mask = mask_image
+
+                #color correction for zeroes inpainting
+                p.color_corrections = [processing.setup_color_correction(init_image)]
 
                 print("Smart mode: inpainting border")
 
                 processed = processing.process_images(p)
                 init_image = processed.images[0].convert('RGB')
-                p = reset_pipeline(root, args, prompt, negative_prompt)
-                p.image_mask = None
-                processed = None
+
+                p = reset_pipeline(args, prompt, negative_prompt) # This should reset as to not lose prompts and base args in next pass
+                p.init_images = [init_image] # preserve the init image that we just generated, as we reset the p object
+
+                processed = None # This needs to be none so that the normal pass will continue
+                mask_image = None # Could be using a standard mask in addition to this pass, so this needs to be reset also
+
+                # Below are the settings that started stacking up
+                #p.inpainting_mask_invert = False
+                #p.sd_model=sd_model
+                #p.color_corrections = None
+                #p.image_mask = None
+                #p.inpainting_fill = 1
+                #p.sd_model=sd_model
+
+                # This setting allowed the diffusion to continue however we decided that resetting the p object 
+                # was safer
+                #p.mask = None
             else:
                 # fix tqdm total steps if we don't have to conduct a second pass
                 tqdm_instance = shared.total_tqdm

From 442e6496afb1309053b5c61132b23ef773d2dee4 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Wed, 28 Dec 2022 02:15:18 +0300
Subject: [PATCH 15/19] remove fill assignment in reset

---
 scripts/deforum_helpers/generate.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index 6c4ceb742..e24278652 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -142,7 +142,6 @@ def reset_pipeline(args, prompt, negative_prompt):
     p.subseed_strength = args.subseed_strength
     p.seed_resize_from_w = args.seed_resize_from_w
     p.seed_resize_from_h = args.seed_resize_from_h
-    p.fill = args.fill
     p.ddim_eta = args.ddim_eta
 
     # Below settings which have conditions or symbols that dont work in the p object constructor

From c9904db37c4e2d55c1cfed8fd671f4e4c9603828 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Sun, 12 Feb 2023 23:29:39 +0300
Subject: [PATCH 16/19] fixup the rest of merge errors

---
 scripts/deforum_helpers/generate.py | 3 +--
 scripts/deforum_helpers/render.py   | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index a4da3667e..a539fe86b 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -174,8 +174,7 @@ def generate(args, anim_args, loop_args, root, frame = 0, return_sample=False, s
             mask = prepare_mask(mask_image, 
                                     (args.W, args.H), 
                                     args.mask_contrast_adjust, 
-                                    args.mask_brightness_adjust, 
-                                    invert_mask=False)
+                                    args.mask_brightness_adjust)
             
             # HACK: this is a hacky check to make the mask work with the new inpainting code
             crop_region = masking.get_crop_region(np.array(mask_image), args.full_res_mask_padding)
diff --git a/scripts/deforum_helpers/render.py b/scripts/deforum_helpers/render.py
index 7900dc3df..896a324d6 100644
--- a/scripts/deforum_helpers/render.py
+++ b/scripts/deforum_helpers/render.py
@@ -220,9 +220,9 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, animat
                     depth = depth_model.predict(turbo_next_image, anim_args, root.half_precision)
                 
                 if advance_prev:
-                    turbo_prev_image, _ = anim_frame_warp(turbo_prev_image, args, anim_args, keys, tween_frame_idx, depth_model, depth=depth, device=root.device, half_precision=root.half_precision)
+                    turbo_prev_image, _, _ = anim_frame_warp(turbo_prev_image, args, anim_args, keys, tween_frame_idx, depth_model, depth=depth, device=root.device, half_precision=root.half_precision)
                 if advance_next:
-                    turbo_next_image, _ = anim_frame_warp(turbo_next_image, args, anim_args, keys, tween_frame_idx, depth_model, depth=depth, device=root.device, half_precision=root.half_precision)
+                    turbo_next_image, _, _ = anim_frame_warp(turbo_next_image, args, anim_args, keys, tween_frame_idx, depth_model, depth=depth, device=root.device, half_precision=root.half_precision)
 
                 # hybrid video motion - warps turbo_prev_image or turbo_next_image to match motion
                 if tween_frame_idx > 0:

From afa49996ffc13d8d088bffbf5a681cdae70cf05b Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Sun, 12 Feb 2023 23:40:06 +0300
Subject: [PATCH 17/19] add smart_border_fill_mode to args

---
 scripts/deforum_helpers/args.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py
index 5830c4060..3660783e7 100644
--- a/scripts/deforum_helpers/args.py
+++ b/scripts/deforum_helpers/args.py
@@ -629,6 +629,9 @@ def show_vid():
                 with gr.Row():
                     choice = mask_fill_choices[d.fill]
                     fill = gr.Radio(label='Mask fill', choices=mask_fill_choices, value=choice, type="index")
+                with gr.Row():
+                    choice = mask_fill_choices[d.smart_border_fill_mode]
+                    smart_border_fill_mode = gr.Radio(label='Smart border fill', choices=mask_fill_choices, value=choice, type="index")
                 with gr.Row():
                     full_res_mask = gr.Checkbox(label="Full res mask", value=d.full_res_mask, interactive=True)
                     full_res_mask_padding = gr.Slider(minimum=0, maximum=512, step=1, label="Full res mask padding", value=d.full_res_mask_padding, interactive=True)
@@ -967,7 +970,7 @@ def change_hybrid_tab_status(choice):
                         use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image,
                         use_mask, use_alpha_as_mask, invert_mask, overlay_mask,
                         mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur,
-                        fill, full_res_mask, full_res_mask_padding,
+                        fill, smart_border_fill_mode, full_res_mask, full_res_mask_padding,
                         reroll_blank_frames'''
                     ).replace("\n", "").replace("\r", "").replace(" ", "").split(',')
 video_args_names =  str(r'''skip_video_for_run_all,

From 305734cdcc84d06625434844bae3693859de50d7 Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Sun, 12 Feb 2023 23:53:00 +0300
Subject: [PATCH 18/19] fix smart mode for 3d

---
 scripts/deforum_helpers/generate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py
index a539fe86b..ed9aa2c1c 100644
--- a/scripts/deforum_helpers/generate.py
+++ b/scripts/deforum_helpers/generate.py
@@ -165,7 +165,7 @@ def generate(args, anim_args, loop_args, root, frame = 0, return_sample=False, s
                 warp_mask_image = Image.fromarray(root.warp_mask).convert('L')
                 for x in range(mask_image.width):
                     for y in range(mask_image.height):
-                        if mask_image.getpixel((x,y)) > 0 or warp_mask_image.getpixel((x,y)) > 0:
+                        if mask_image.getpixel((x,y)) > 0 or warp_mask_image.getpixel((x,y)) == 0:
                             mask_image.putpixel((x,y), 255)
                         else:
                             mask_image.putpixel((x,y), 0)

From 01b1e9c31a7dc6a6368b7b82b858518fd841514f Mon Sep 17 00:00:00 2001
From: kabachuha <artemkhrapov2001@yandex.ru>
Date: Mon, 13 Feb 2023 00:19:06 +0300
Subject: [PATCH 19/19] 2d fixup

---
 scripts/deforum_helpers/animation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/deforum_helpers/animation.py b/scripts/deforum_helpers/animation.py
index 8b5d5344e..b04383b30 100644
--- a/scripts/deforum_helpers/animation.py
+++ b/scripts/deforum_helpers/animation.py
@@ -184,7 +184,7 @@ def anim_frame_warp_2d(prev_img_cv2, args, anim_args, keys, frame_idx):
     rot_mat = np.vstack([rot_mat, [0,0,1]])
     if anim_args.enable_perspective_flip:
         bM = get_flip_perspective_matrix(args.W, args.H, keys, frame_idx)
-        rot_mat = np.matmul(bM, rot_mat, trans_mat)
+        xform = np.matmul(bM, rot_mat, trans_mat)
     else:
         xform = np.matmul(rot_mat, trans_mat)