M7: Whisper full-checkpoint conformance (openai/whisper-tiny)

Full-size Whisper-tiny forward pass on Emily.Backend, pinned
against a deterministic synthetic 30 s mel window (sin(iota({1,
3000, 80}) * 0.01)) and a short special-token decoder prompt.
Asserts the leading 3×3 logits slice at 1e-3 tolerance plus the
decoder-last-step argmax (== 50257, <|endoftext|>, which is what
the synthetic input collapses to — that's fine for a pin; the
assertion is "same backend + same weights + same input reproduces
the same token", not "the model says something interesting").

Runtime is ~7 minutes on Apple Silicon because Emily.Backend.conv
still routes through Nx.BinaryBackend (deferred to M8). Whisper
encoder has two 1-D convs (kernel=3, out=384) over 3000 time-steps,
~940M multiply-adds executed as BEAM loops. The full test is
:whisper_full tagged, excluded from --only conformance, and opt-in
only; M8 wiring Backend.conv to Native.conv_general will bring this
to sub-second.

Also picks up formatter fixups on lib/emily/backend.ex (the
via_binary assignment from the previous commit) and trailing blank
lines on distilbert_test.exs / qwen3_test.exs.

Author: ausimian

lib/emily/backend.ex

@@ -966,9 +966,10 @@ defmodule Emily.Backend do
   # `Emily.Backend` during conformance tests — and the resulting
   # mixed-backend operand list crashes inside BinaryBackend's op.
   defp via_binary(%T{} = out, tensors, fun) when is_list(tensors) do
-    result = Nx.with_default_backend(Nx.BinaryBackend, fn ->
-      tensors |> transfer_all() |> then(&apply(fun, &1))
-    end)
+    result =
+      Nx.with_default_backend(Nx.BinaryBackend, fn ->
+        tensors |> transfer_all() |> then(&apply(fun, &1))
+      end)
 
     from_binary(out, Nx.to_binary(result), [])
   end

test/emily/conformance/distilbert_test.exs

@@ -203,5 +203,4 @@ defmodule Emily.Conformance.DistilbertTest do
       end
     end
   end
-
 end

test/emily/conformance/qwen3_test.exs

@@ -180,5 +180,4 @@ defmodule Emily.Conformance.Qwen3Test do
       Enum.take(tokens, len_val)
     end
   end
-
 end

test/emily/conformance/whisper_full_test.exs

@@ -0,0 +1,91 @@
+defmodule Emily.Conformance.WhisperFullTest do
+  @moduledoc """
+  Full `openai/whisper-tiny` end-to-end conformance test.
+
+  Like the other `*_full` suites, this is excluded even from
+  `mix test --only conformance`: the model is ~150 MB on first
+  fetch. Run explicitly:
+
+      mix test --only whisper_full
+
+  Uses a deterministic synthetic mel-features tensor rather than a
+  checked-in audio fixture, by the same reasoning as
+  `vit_full_test.exs`: binary test assets in git are annoying, and
+  the intent is to catch numerical drift on real-size weight
+  tensors (encoder conv frontend + 4 encoder blocks + 4 decoder
+  blocks with cross-attention), not to verify mel-spectrogram
+  computation. Input shape is Whisper's canonical 30-second window:
+  3000 time-steps × 80 mel bins.
+
+  A failure means the backend has drifted, Bumblebee's Whisper port
+  has changed, or the HF checkpoint has been republished — all of
+  which are real signals.
+  """
+
+  use ExUnit.Case, async: false
+  use Emily.ConformanceHelper
+
+  @moduletag :whisper_full
+  @moduletag capture_log: true
+  @moduletag timeout: 600_000
+
+  test "openai/whisper-tiny forward pass matches pinned logits slice" do
+    {:ok, %{model: model, params: params, spec: spec}} =
+      Bumblebee.load_model({:hf, "openai/whisper-tiny"})
+
+    assert %Bumblebee.Audio.Whisper{architecture: :for_conditional_generation} = spec
+
+    # Synthetic 30 s mel window. Nx.iota + sin produces a
+    # fully-deterministic, feature-rich signal — more useful than
+    # a constant because it exercises the attention pattern rather
+    # than landing on a degenerate uniform hidden state.
+    input_features =
+      Nx.sin(Nx.iota({1, 3000, 80}, type: :f32) |> Nx.multiply(0.01))
+
+    # Short decoder prompt: the four Whisper special tokens that open
+    # every English-language transcription (<|startoftranscript|>,
+    # <|en|>, <|transcribe|>, <|notimestamps|>) plus two text-token
+    # placeholders. Exact ids don't matter for a numerical pin —
+    # they just need to be in-vocab and deterministic.
+    decoder_input_ids = Nx.tensor([[50_258, 50_259, 50_359, 50_363, 50, 100]])
+    decoder_attention_mask = Nx.tensor([[1, 1, 1, 1, 1, 1]])
+
+    inputs = %{
+      "input_features" => input_features,
+      "decoder_input_ids" => decoder_input_ids,
+      "decoder_attention_mask" => decoder_attention_mask
+    }
+
+    outputs = Axon.predict(model, params, inputs)
+
+    assert Nx.shape(outputs.logits) == {1, 6, 51_865}
+
+    argmax =
+      outputs.logits[[.., -1, ..]]
+      |> Nx.argmax(axis: -1)
+      |> Nx.backend_transfer(Nx.BinaryBackend)
+      |> Nx.to_flat_list()
+      |> hd()
+
+    # On the synthetic mel input the decoder collapses to
+    # <|endoftext|> (50257) by the last position. That's fine for a
+    # pin — the assertion is "the same backend + the same weights +
+    # the same input reproduce the same token", not "the model says
+    # something interesting".
+    assert argmax == 50_257
+
+    assert_all_close(
+      outputs.logits[[.., 0..2, 0..2]],
+      Nx.tensor([
+        [[2.9246, 0.2663, 3.8530], [-4.5523, -8.4833, -4.4232], [17.7350, 16.3070, 13.2149]]
+      ]),
+      # Whisper-tiny under BinaryBackend fallback for conv produces
+      # slightly different rounding than a pure Emily path would, so
+      # loosen the numerical tolerance marginally relative to the
+      # tiny-random suite. Still three orders of magnitude inside
+      # f16/bf16 accumulation drift territory.
+      atol: 1.0e-3,
+      rtol: 1.0e-3
+    )
+  end
+end

test/emily/conformance/whisper_test.exs

@@ -71,7 +71,7 @@ defmodule Emily.Conformance.WhisperTest do
 
     outputs = Axon.predict(model, params, inputs)
 
-    assert Nx.shape(outputs.logits) == {1, 8, 50257}
+    assert Nx.shape(outputs.logits) == {1, 8, 50_257}
 
     assert_all_close(
       outputs.logits[[.., 1..3, 1..3]],