Experimental render core with keyframe redistribution logic for Parseq.

scripts/default_settings.txt

@@ -143,7 +143,8 @@
     "hybrid_comp_save_extra_frames": false,
     "parseq_manifest": "",
     "parseq_use_deltas": true,
-    "parseq_non_schedule_overrides": true,    
+    "parseq_non_schedule_overrides": true,
+    "parseq_key_frame_redistribution": "off",
     "use_looper": false,
     "init_images": "{\n    \"0\": \"https://deforum.github.io/a1/Gi1.png\",\n    \"max_f/4-5\": \"https://deforum.github.io/a1/Gi2.png\",\n    \"max_f/2-10\": \"https://deforum.github.io/a1/Gi3.png\",\n    \"3*max_f/4-15\": \"https://deforum.github.io/a1/Gi4.jpg\",\n    \"max_f-20\": \"https://deforum.github.io/a1/Gi1.png\"\n}",
     "image_strength_schedule": "0:(0.75)",

scripts/deforum_helpers/args.py

@@ -22,7 +22,8 @@
 import modules.paths as ph
 import modules.shared as sh
 from modules.processing import get_fixed_seed
-from .defaults import get_guided_imgs_default_json, mask_fill_choices, get_samplers_list, get_schedulers_list
+from .defaults import (get_guided_imgs_default_json, get_parseq_keyframe_redistributions_list,
+                       get_samplers_list, get_schedulers_list)
 from .deforum_controlnet import controlnet_component_names
 from .general_utils import get_os, substitute_placeholders
 
@@ -928,7 +929,7 @@ def DeforumArgs():
             "type": "checkbox",
             "value": False,
             "info": "Preview motion only. Uses a static picture for init, and draw motion reference rectangle."
-        },        
+        },
     }
 
 def LoopArgs():
@@ -990,7 +991,14 @@ def ParseqArgs():
             "type": "checkbox",
             "value": True,
             "info": "Recommended. If you uncheck this, the FPS, max_frames and cadence in the Parseq doc are ignored, and the values in the A1111 UI are used instead."
-        }        
+        },
+        "parseq_key_frame_redistribution": {
+            "label": "Parseq key frame redistribution.",
+            "type": "dropdown",
+            "choices": get_parseq_keyframe_redistributions_list().values(),
+            "value": "None",
+            "info": "Gain Parseq precision at the cost of cadence regularity. Allows for fast generations at high cadence."
+        },
     }
 
 def DeforumOutputArgs():
@@ -1119,8 +1127,7 @@ def DeforumOutputArgs():
             "value": False,
             "info": "Interpolate upscaled images, if available",
             "visible": False
-        },        
-
+        },
     }
 
 def get_component_names():

scripts/deforum_helpers/defaults.py

@@ -47,6 +47,13 @@ def get_schedulers_list():
         'sgm uniform': 'SGM Uniform'
     }
 
+def get_parseq_keyframe_redistributions_list():
+    return {
+        'off': 'Off',
+        'parseq_only': 'Parseq Only (no cadence)',
+        'uniform_with_parseq': 'Uniform with Parseq (pseudo-cadence)'
+    }
+
 def DeforumAnimPrompts():
     return r"""{
     "0": "tiny cute bunny, vibrant diffraction, highly detailed, intricate, ultra hd, sharp photo, crepuscular rays, in focus",

scripts/deforum_helpers/render.py

@@ -48,11 +48,18 @@
 from .prompt import prepare_prompt
 from modules.shared import opts, cmd_opts, state, sd_model
 from modules import lowvram, devices, sd_hijack
+from .rendering import experimental_core
 from .RAFT import RAFT
 
 from deforum_api import JobStatusTracker
 
 def render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root):
+    is_use_parseq = parseq_args.parseq_manifest and parseq_args.parseq_manifest.strip()
+    is_use_key_frame_redistribution = parseq_args.parseq_key_frame_redistribution != "Off"
+    is_use_new_render_core = is_use_parseq and is_use_key_frame_redistribution
+    if is_use_new_render_core:
+        experimental_core.render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root)
+        return
 
     # initialise Parseq adapter
     parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args, loop_args)
@@ -538,7 +545,7 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, contro
             devices.torch_gc()
             lowvram.setup_for_low_vram(sd_model, cmd_opts.medvram)
             sd_hijack.model_hijack.hijack(sd_model)
-        
+
         optical_flow_redo_generation = anim_args.optical_flow_redo_generation if not args.motion_preview_mode else 'None'
 
         # optical flow redo before generation
@@ -637,7 +644,7 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, contro
 
         args.seed = next_seed(args, root)
 
-        last_preview_frame = render_preview(args, anim_args, video_args, root, frame_idx, last_preview_frame)            
+        last_preview_frame = render_preview(args, anim_args, video_args, root, frame_idx, last_preview_frame)
 
         JobStatusTracker().update_phase(root.job_id, phase="GENERATING", progress=frame_idx/anim_args.max_frames)
 
@@ -647,4 +654,3 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, contro
 
     if load_raft:
         raft_model.delete_model()
-

scripts/deforum_helpers/rendering/data/__init__.py

@@ -0,0 +1,6 @@
+from .images import Images
+from .indexes import Indexes
+from .mask import Mask
+from .schedule import Schedule
+from .render_data import RenderData
+from .turbo import Turbo

scripts/deforum_helpers/rendering/data/anim/__init__.py

@@ -0,0 +1,2 @@
+from .animation_keys import AnimationKeys
+from .animation_mode import AnimationMode

scripts/deforum_helpers/rendering/data/anim/animation_keys.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass
+
+from ....animation_key_frames import DeformAnimKeys, LooperAnimKeys
+
+
+@dataclass(init=True, frozen=True, repr=False, eq=False)
+class AnimationKeys:
+    deform_keys: DeformAnimKeys
+    looper_keys: LooperAnimKeys
+
+    @staticmethod
+    def _choose_default_or_parseq_keys(default_keys, parseq_keys, parseq_adapter):
+        return default_keys if not parseq_adapter.use_parseq else parseq_keys
+
+    @staticmethod
+    def from_args(step_args, parseq_adapter, seed):
+        ada = parseq_adapter
+
+        def _choose(default_keys):
+            return AnimationKeys._choose_default_or_parseq_keys(default_keys, ada.anim_keys, ada)
+
+        # Parseq keys are decorated, see ParseqAnimKeysDecorator and ParseqLooperKeysDecorator
+        return AnimationKeys(_choose(DeformAnimKeys(step_args.anim_args, seed)),
+                             _choose(LooperAnimKeys(step_args.loop_args, step_args.anim_args, seed)))

scripts/deforum_helpers/rendering/data/anim/animation_mode.py

@@ -0,0 +1,95 @@
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from ...util import opt_utils
+from ....RAFT import RAFT
+from ....hybrid_video import hybrid_generation
+
+
+@dataclass(init=True, frozen=False, repr=False, eq=False)
+class AnimationMode:
+    has_video_input: bool = False
+    hybrid_input_files: Any = None
+    hybrid_frame_path: str = ""
+    prev_flow: Any | None = None
+    is_keep_in_vram: bool = False
+    depth_model: Any = None
+    raft_model: Any = None
+
+    def is_predicting_depths(self) -> bool:
+        return self.depth_model is not None
+
+    def is_raft_active(self) -> bool:
+        return self.raft_model is not None
+
+    def unload_raft_and_depth_model(self):
+        if self.is_predicting_depths() and not self.is_keep_in_vram:
+            self.depth_model.delete_model()  # handles adabins too
+        if self.is_raft_active():
+            self.raft_model.delete_model()
+
+    @staticmethod
+    def _has_video_input(anim_args) -> bool:
+        return AnimationMode._is_2d_or_3d_mode(anim_args) and AnimationMode._is_using_hybrid_frames(anim_args)
+
+    @staticmethod
+    def _is_2d_or_3d_mode(anim_args):
+        return anim_args.animation_mode in ['2D', '3D']
+
+    @staticmethod
+    def _is_using_hybrid_frames(anim_args):
+        return (anim_args.hybrid_composite != 'None'
+                or anim_args.hybrid_motion in ['Affine', 'Perspective', 'Optical Flow'])
+
+    @staticmethod
+    def _is_requiring_hybrid_frames(anim_args):
+        return AnimationMode._is_2d_or_3d_mode(anim_args) and AnimationMode._is_using_hybrid_frames(anim_args)
+
+    @staticmethod
+    def _is_load_depth_model_for_3d(args, anim_args):
+        is_depth_warped_3d = anim_args.animation_mode == '3D' and anim_args.use_depth_warping
+        has_depth_or_depth_video_mask = anim_args.hybrid_comp_mask_type in ['Depth', 'Video Depth']
+        is_composite_with_depth_mask = anim_args.hybrid_composite and has_depth_or_depth_video_mask
+        is_depth_used = is_depth_warped_3d or anim_args.save_depth_maps or is_composite_with_depth_mask
+        return is_depth_used and not args.motion_preview_mode
+
+    @staticmethod
+    def load_raft_if_active(anim_args, args):
+        is_cadenced_raft = anim_args.optical_flow_cadence == "RAFT" and int(anim_args.diffusion_cadence) > 1
+        is_optical_flow_raft = anim_args.hybrid_motion == "Optical Flow" and anim_args.hybrid_flow_method == "RAFT"
+        is_raft_redo = anim_args.optical_flow_redo_generation == "RAFT"
+        is_load_raft = (is_cadenced_raft or is_optical_flow_raft or is_raft_redo) and not args.motion_preview_mode
+        if is_load_raft:
+            print("Loading RAFT model...")
+        return RAFT() if is_load_raft else None
+
+    @staticmethod
+    def load_depth_model_if_active(args, anim_args):
+        return AnimationMode._is_load_depth_model_for_3d(args, anim_args) \
+            if opt_utils.keep_3d_models_in_vram() else None
+
+    @staticmethod
+    def initial_hybrid_files(sa) -> list[Path]:
+        """Returns a list of initial hybrid input files if required, otherwise an empty list."""
+        if AnimationMode._is_requiring_hybrid_frames(sa.anim_args):
+            # may cause side effects on args and anim_args.
+            _, __, init_hybrid_input_files = hybrid_generation(sa.args, sa.anim_args, sa.root)
+            return init_hybrid_input_files
+        return []
+
+    @staticmethod
+    def from_args(step_args):
+        sa = step_args  # RenderInitArgs
+        # path required by hybrid functions, even if hybrid_comp_save_extra_frames is False
+        hybrid_input_files: Any = os.path.join(sa.args.outdir, 'hybridframes')
+        previous_flow = None
+        return AnimationMode(
+            AnimationMode._has_video_input(sa.anim_args),
+            AnimationMode.initial_hybrid_files(sa),
+            hybrid_input_files,
+            previous_flow,
+            opt_utils.keep_3d_models_in_vram(),
+            AnimationMode.load_depth_model_if_active(sa.args, sa.anim_args),
+            AnimationMode.load_raft_if_active(sa.anim_args, sa.args))

scripts/deforum_helpers/rendering/data/frame/__init__.py

@@ -0,0 +1,4 @@
+from .key_frame_distribution import KeyFrameDistribution
+from .key_frame_data import KeyFrameData
+from .key_frame import KeyFrame
+from .tween_frame import Tween