Re-order actions enum

src/action.nim

@@ -7,9 +7,9 @@ template writeAt(baseBuffer: auto, index: int, offset: int, val: untyped) =
 
 type
   ActionKind* = enum
-    actSpeed, actVarispeed, actVolume, actDeesser, actInvert, actZoom, actHflip,
-    actVflip, actOpacity, actBlur, actBrightness, actLuv, actLens, actRotate,
-    actDrawbox, actPos, actSpin, actColorKey, actLoop, actChromaKey
+    actSpeed, actVarispeed, actVolume, actDeesser, actInvert, actHflip, actVflip,
+    actZoom, actOpacity, actBlur, actBrightness, actLuv, actLens, actRotate, actSpin,
+    actDrawbox, actPos, actColorKey, actChromaKey, actLoop
 
   Easing* = enum  # interpolation curve for animations
     easeLinear, easeIn, easeOut, easeInOut
@@ -147,9 +147,9 @@ Positional args: `intensity` sets how much to de-ess (0.0 = none, 1.0 = maximum)
 Distort the picture like a camera lens. With no arguments, a fun fisheye is applied. Implemented via ffmpeg's `lenscorrection` filter.
 Positional args: `k1` is the quadratic correction factor and `k2` the double-quadratic factor. Negative values bulge the image outward (fisheye); positive values pinch it inward (pincushion)."""),
   ActionDef(name: "colorkey", flags: {afVideo}, argSpec: "color[:similar:blend]", range: rng(0.0, 1.0),
-    help: "Make a color transparent by matching it in RGB space. Best for flat, synthetic backgrounds (a logo's matte, a screen recording, a gif with one clean color); for real green-/blue-screen camera footage use `chromakey` instead. Positional args: `color` is the key color (a name like `green` or a hex value), `similar` how close a pixel must be to be keyed (default 0.01), and `blend` how soft the edge is (default 0.0). Implemented via ffmpeg's `colorkey` filter."),
+    help: "Make a color transparent by matching it in RGB space. Best for flat, synthetic backgrounds (a logo's matte, a screen recording, a gif with one clean color); for real green-/blue-screen camera footage use `chromakey` instead. On the base (bottom) video track there is nothing to reveal, so the matched color is replaced with the timeline background (`-bg`) instead. Positional args: `color` is the key color (a name like `green` or a hex value), `similar` how close a pixel must be to be keyed (default 0.25), and `blend` how soft the edge is (default 0.0). Implemented via ffmpeg's `colorkey` filter."),
   ActionDef(name: "chromakey", flags: {afVideo}, argSpec: "color[:similar:blend]", range: rng(0.0, 1.0),
-    help: "Make a color transparent by matching it in chroma (YUV) space, tolerating lighting variation, shadows, and soft edges. This is the green-/blue-screen keyer for real camera footage; for flat synthetic backgrounds use `colorkey` instead. Positional args: `color` is the key color (a name like `green` or a hex value), `similar` how close a pixel must be to be keyed (default 0.01), and `blend` how soft the edge is (default 0.0). Implemented via ffmpeg's `chromakey` filter."),
+    help: "Make a color transparent by matching it in chroma (YUV) space, tolerating lighting variation, shadows, and soft edges. This is the green-/blue-screen keyer for real camera footage; for flat synthetic backgrounds use `colorkey` instead. On the base (bottom) video track there is nothing to reveal, so the matched color is replaced with the timeline background (`-bg`) instead. Positional args: `color` is the key color (a name like `green` or a hex value), `similar` how close a pixel must be to be keyed (default 0.25), and `blend` how soft the edge is (default 0.0). Implemented via ffmpeg's `chromakey` filter."),
   ActionDef(name: "loop", flags: {afVideo},
     help: "Loop the clip's source back to its start when it runs out of frames, instead of ending. Useful for overlays whose source (e.g. a short gif) is shorter than the section it covers, e.g. `add:logo.gif,loop`."),
 ]
@@ -340,7 +340,7 @@ func parseAction*(val: string): Action {.raises: [ActionParseError].} =
       except ValueError:
         raise newException(ActionParseError, "Invalid color: " & parts[1])
     )
-    var vals = [toUnorm16(0.01'f32), toUnorm16(0.0'f32)]
+    var vals = [toUnorm16(0.25'f32), toUnorm16(0.0'f32)]
     for idx in 2 ..< parts.len:
       vals[idx - 2] = (
         try:

src/render/video.nim

@@ -500,6 +500,36 @@ proc makeNewVideoFrames*(output: var OutputContainer, tl: v3, args: mainArgs,
       else:
         hi = mid - 1
 
+  proc keyOverBg(frame0: ptr AVFrame, effect: Action): ptr AVFrame =
+    var frame = frame0
+    let w = frame.width
+    let h = frame.height
+    let col = effect.color.toString
+    let isChroma = effect.kind == actChromaKey
+    var bgFrame = makeSolid(w, h, tl.bg)
+    frame.pts = 0
+    bgFrame.pts = 0
+    let g = newGraph()
+    let bgSrc = g.add("buffer", &"video_size={w}x{h}:pix_fmt=yuv420p:time_base={graphTb}:pixel_aspect=1/1")
+    let fgSrc = g.add("buffer", &"video_size={w}x{h}:pix_fmt={$AVPixelFormat(frame.format)}:time_base={graphTb}:pixel_aspect=1/1")
+    let toAlpha = g.add("format", "pix_fmts=" & (if isChroma: "yuva420p" else: "rgba"))
+    let keyer = g.add((if isChroma: "chromakey" else: "colorkey"),
+      &"{col}:{effect.similar}:{effect.blend}")
+    let ov = g.add("overlay", "format=yuv420")
+    discard g.linkNodes(@[fgSrc, toAlpha, keyer])
+    g.link(bgSrc, ov, 0, 0)  # background on the bottom pad
+    g.link(keyer, ov, 0, 1)  # keyed frame on top
+    g.link(ov, g.add("buffersink"))
+    g.configure()
+    g.pushIdx(0, bgFrame)
+    g.pushIdx(1, frame)
+    g.flushIdx(0)
+    g.flushIdx(1)
+    result = g.pull()
+    g.cleanup()
+    av_frame_free(addr bgFrame)
+    av_frame_free(addr frame)
+
   proc applyEffects(frame0: ptr AVFrame, effects: Actions, local, clipDur: int,
       isOverlay = false): ptr AVFrame =
     ## Apply one clip's effect chain to a frame, returning the (possibly new)
@@ -740,44 +770,32 @@ proc makeNewVideoFrames*(output: var OutputContainer, tl: v3, args: mainArgs,
         fxGraph.push(frame)
         av_frame_free(addr frame)
         frame = fxGraph.pull()
-      of actColorKey:
-        if not isOverlay:
-          continue
-        let col = effect.color.toString
-        let frameFmtName = $AVPixelFormat(frame.format)
-        let bufferArgs = &"video_size={frame.width}x{frame.height}:pix_fmt={frameFmtName}:time_base={graphTb}:pixel_aspect=1/1"
-        let key = &"colorkey|{col}|{effect.similar}|{effect.blend}|{bufferArgs}"
-        if fxKey != key:
-          if fxGraph != nil:
-            fxGraph.cleanup()
-          fxGraph = newGraph()
-          let bufferSrc = fxGraph.add("buffer", bufferArgs)
-          let clrkey = fxGraph.add("colorkey", &"{col}:{effect.similar}:{effect.blend}")
-          let bufferSink = fxGraph.add("buffersink")
-          fxGraph.linkNodes(@[bufferSrc, clrkey, bufferSink]).configure()
-          fxKey = key
-        fxGraph.push(frame)
-        av_frame_free(addr frame)
-        frame = fxGraph.pull()
-      of actChromaKey:
+      of actColorKey, actChromaKey:
         if not isOverlay:
+          # Base layer: no lower track to reveal, so replace the keyed color with
+          # the timeline background instead of making it transparent.
+          frame = keyOverBg(frame, effect)
           continue
         let col = effect.color.toString
         let frameFmtName = $AVPixelFormat(frame.format)
         let bufferArgs = &"video_size={frame.width}x{frame.height}:pix_fmt={frameFmtName}:time_base={graphTb}:pixel_aspect=1/1"
-        let key = &"chromakey|{col}|{effect.similar}|{effect.blend}|{bufferArgs}"
+        let key = &"{effect.kind}|{col}|{effect.similar}|{effect.blend}|{bufferArgs}"
         if fxKey != key:
           if fxGraph != nil:
             fxGraph.cleanup()
           fxGraph = newGraph()
           let bufferSrc = fxGraph.add("buffer", bufferArgs)
-          # chromakey keys in YUV-with-alpha; convert in, then back to rgba so the
-          # composited overlay keeps its alpha channel.
-          let toYuva = fxGraph.add("format", "pix_fmts=yuva420p")
-          let chrkey = fxGraph.add("chromakey", &"{col}:{effect.similar}:{effect.blend}")
-          let toRgba = fxGraph.add("format", "pix_fmts=rgba")
-          let bufferSink = fxGraph.add("buffersink")
-          fxGraph.linkNodes(@[bufferSrc, toYuva, chrkey, toRgba, bufferSink]).configure()
+          var nodes = @[bufferSrc]
+          if effect.kind == actChromaKey:
+            # chromakey keys in YUV-with-alpha; convert in, then back to rgba so
+            # the composited overlay keeps its alpha channel.
+            nodes.add fxGraph.add("format", "pix_fmts=yuva420p")
+            nodes.add fxGraph.add("chromakey", &"{col}:{effect.similar}:{effect.blend}")
+            nodes.add fxGraph.add("format", "pix_fmts=rgba")
+          else:
+            nodes.add fxGraph.add("colorkey", &"{col}:{effect.similar}:{effect.blend}")
+          nodes.add fxGraph.add("buffersink")
+          fxGraph.linkNodes(nodes).configure()
           fxKey = key
         fxGraph.push(frame)
         av_frame_free(addr frame)

tests/test.py

@@ -601,6 +601,27 @@ def test_drawbox(self) -> None:
                 assert r > 200 and g < 80 and b < 80, f"box px {(r, g, b)}"
                 break
 
+    def test_colorkey_base(self) -> None:
+        # On the base track, colorkey has nothing to reveal, so the keyed green
+        # must be replaced with the timeline background (white).
+        out = self.main(
+            ["resources/only-video/man-on-green-screen.mp4"],
+            ["-bg", "white", "-e", "none", "-w:1", "colorkey:#14DB00"],
+            "ck.mp4",
+        )
+        with av.open(out) as container:
+            # Frame 25: the figure has walked into the green screen by now.
+            for i, frame in enumerate(container.decode(container.streams.video[0])):
+                if i < 25:
+                    continue
+                rgb = frame.to_ndarray(format="rgb24")
+                r, g, b = rgb[360, 640]  # center: was green, now background
+                assert r > 230 and g > 230 and b > 230, f"center px {(r, g, b)}"
+                # The figure (dark silhouette) must survive the key.
+                dark = (rgb.max(axis=2) < 48).mean()
+                assert dark > 0.05, f"figure keyed away: {dark * 100:.2f}% dark"
+                break
+
     def test_add_overlay(self) -> None:
         png = self.make_png("ov.png", 200, 200, (255, 0, 0))
         out = self.main(