From 87dc6d5ebe60e192288863a6ff99b60c02bc1a8f Mon Sep 17 00:00:00 2001 From: Tong Wu Date: Mon, 3 Aug 2026 13:40:51 -0700 Subject: [PATCH 1/3] Delete four deletion-eligible ALBEF embedding tests (#551) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Oncall test cleanup for the `stl_multimodal` rotation. TestX flags these four tests in `tests/models/albef/test_albef.py` as eligible for deletion — all sit in `DISABLED_FAILING` trunk state with open `FAILURE` issues, well past the 60-day zero-value threshold: - `test_albef_image_embeddings` (`844425197297248`, `844425197101365`) - `test_albef_image_embeddings_momentum` (`562950220612232`, `844425197101313`) - `test_albef_multimodal_embeddings` (`844425197297177`, `844425197101408`) - `test_albef_multimodal_embeddings_momentum` (`562950220612287`, `281475243658978`) Each asserted a hard-coded 2x2x3 activation tensor to `atol=1e-4` against a seeded forward pass — golden values that drift whenever RNG or kernel behavior changes underneath them. Re-baselining numbers nobody has validated since they broke would restore a green signal without restoring any real assurance, so they are deleted instead. The `albef_model_output` fixture and the surrounding tests are untouched: `test_albef_text_embeddings` and `test_albef_text_embeddings_momentum` still exercise the same forward pass, and `test_similarity`, `test_neg_embeddings`, `test_dequeue_and_enqueue`, `test_momentum_update`, and `test_copy_params_momentum_models` still cover `ALBEFModelWithSimilarity`. Differential Revision: D114397069 --- tests/models/albef/test_albef.py | 44 -------------------------------- 1 file changed, 44 deletions(-) diff --git a/tests/models/albef/test_albef.py b/tests/models/albef/test_albef.py index 922712a39..5636a3b94 100644 --- a/tests/models/albef/test_albef.py +++ b/tests/models/albef/test_albef.py @@ -79,26 +79,6 @@ def albef_model_output(albef_model): return albef_model(image, text, text_atts) -def test_albef_image_embeddings(albef_model_output): - expected = Tensor( - [ - [[1.364883, -1.003092, -0.361791], [-0.634884, 1.411830, -0.776947]], - [[1.401580, -0.537510, -0.864071], [1.378901, -0.417473, -0.961429]], - ] - ) - assert_expected(albef_model_output.image_embeddings, expected, rtol=0, atol=1e-4) - - -def test_albef_image_embeddings_momentum(albef_model_output): - expected = Tensor( - [ - [[1.364883, -1.003092, -0.361791], [-0.634884, 1.411830, -0.776947]], - [[1.401580, -0.537510, -0.864070], [1.378902, -0.417473, -0.961429]], - ] - ) - assert_expected(albef_model_output.image_embeddings_m, expected, rtol=0, atol=1e-4) - - def test_albef_text_embeddings(albef_model_output): expected = Tensor( [ @@ -119,30 +99,6 @@ def test_albef_text_embeddings_momentum(albef_model_output): assert_expected(albef_model_output.text_embeddings_m, expected, rtol=0, atol=1e-4) -def test_albef_multimodal_embeddings(albef_model_output): - expected = Tensor( - [ - [[-0.068738, 1.257666, -1.188928], [1.409873, -0.609056, -0.800817]], - [[-1.402520, 0.544084, 0.858435], [1.202279, -1.246038, 0.043760]], - ] - ) - assert_expected( - albef_model_output.multimodal_embeddings, expected, rtol=0, atol=1e-4 - ) - - -def test_albef_multimodal_embeddings_momentum(albef_model_output): - expected = Tensor( - [ - [[-0.068738, 1.257666, -1.188928], [1.409873, -0.609056, -0.800817]], - [[-1.402520, 0.544084, 0.858435], [1.202279, -1.246038, 0.043760]], - ] - ) - assert_expected( - albef_model_output.multimodal_embeddings_m, expected, rtol=0, atol=1e-4 - ) - - def test_copy_params_momentum_models(): model = nn.Linear(3, 2) model_m = copy.deepcopy(model) From 4674d73ae125f8078b0c6fdd80762005dafbd6f6 Mon Sep 17 00:00:00 2001 From: Tong Wu Date: Mon, 3 Aug 2026 13:40:51 -0700 Subject: [PATCH 2/3] Delete deletion-eligible ALBEF test_vision_transformer (#552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Oncall test cleanup for the `stl_multimodal` rotation. TestX flags `TestALBEFVisionEncoder.test_vision_transformer` (test IDs `281475243855620` and `281475243658858`) as eligible for deletion — it sits in `DISABLED_FAILING` trunk state with an open `FAILURE` issue, well past the 60-day zero-value threshold. It was another seeded golden-value test, asserting a hard-coded 1x2x3 tensor to `atol=1e-4`, so it is deleted rather than re-baselined. `assert_expected` and `Tensor` were only used by the deleted test, so both imports are dropped. `set_rng_seed` stays — it is still called at class scope. The four negative tests remain and are the more durable half of this file: `test_invalid_input_length`, `test_invalid_image_channel_dim`, `test_invalid_image_height`, and `test_invalid_image_width` still pin `ALBEFVisionEncoder` input validation. Differential Revision: D114397059 --- tests/models/albef/test_image_encoder.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tests/models/albef/test_image_encoder.py b/tests/models/albef/test_image_encoder.py index df580c2a2..74524805d 100644 --- a/tests/models/albef/test_image_encoder.py +++ b/tests/models/albef/test_image_encoder.py @@ -7,8 +7,7 @@ import pytest import torch -from tests.test_utils import assert_expected, set_rng_seed -from torch import Tensor +from tests.test_utils import set_rng_seed from torchmultimodal.models.albef.image_encoder import ALBEFVisionEncoder @@ -24,19 +23,6 @@ class TestALBEFVisionEncoder: mlp_dim=6, ) - def test_vision_transformer(self): - set_rng_seed(0) - vit = self.vision_encoder - input = torch.randn(1, 3, 4, 4) - output = vit(input) - expected = Tensor( - [ - [1.399478, -0.875986, -0.523492], - [-0.869867, 1.400589, -0.530722], - ] - ).unsqueeze(0) - assert_expected(output, expected, rtol=0, atol=1e-4) - def test_invalid_input_length(self): input = torch.randn(3, 4, 4) with pytest.raises(IndexError, match="index out of range"): From 4ba44637b6f125bb8376a911aa4eb0b736571bda Mon Sep 17 00:00:00 2001 From: Tong Wu Date: Mon, 3 Aug 2026 13:40:51 -0700 Subject: [PATCH 3/3] Delete three deletion-eligible omnivore forward tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Oncall test cleanup for the `stl_multimodal` rotation. TestX flags these three tests in `tests/models/test_omnivore.py` as eligible for deletion — all sit in `DISABLED_FAILING` trunk state with open `FAILURE` issues, well past the 60-day zero-value threshold: - `test_omnivore_swin_t_forward` (`562950220612371`, `844425197101258`) - `test_omnivore_swin_s_forward` (`562950220612245`, `281475243658871`) - `test_omnivore_swin_b_forward` (`281475243855615`, `844425197101257`) Each asserted hard-coded output magnitudes (for example `184.01417`, `239.73104`, `278.06488`) from a seeded forward pass across the image, rgbd, and video heads. These are golden values with no independently derived expectation, so they are deleted rather than re-baselined. The `omnivore_swin_s_model` and `omnivore_swin_b_model` fixtures existed only to serve the deleted tests and are removed with them. `assert_expected` was likewise only used by the deleted tests, so that import is dropped; `set_rng_seed` stays because the `device` fixture still calls it. `test_omnivore_forward_wrong_input_type` remains and still covers the `input_type` validation path via `omnivore_swin_t_model`. Differential Revision: D114397066 --- tests/models/test_omnivore.py | 84 +---------------------------------- 1 file changed, 1 insertion(+), 83 deletions(-) diff --git a/tests/models/test_omnivore.py b/tests/models/test_omnivore.py index 1134d20da..bda5aeac8 100644 --- a/tests/models/test_omnivore.py +++ b/tests/models/test_omnivore.py @@ -7,7 +7,7 @@ import pytest import torch import torchmultimodal.models.omnivore as omnivore -from tests.test_utils import assert_expected, set_rng_seed +from tests.test_utils import set_rng_seed from torchmultimodal.utils.common import get_current_device @@ -22,88 +22,6 @@ def omnivore_swin_t_model(device): return omnivore.omnivore_swin_t().to(device) -@pytest.fixture() -def omnivore_swin_s_model(device): - return omnivore.omnivore_swin_s().to(device) - - -@pytest.fixture() -def omnivore_swin_b_model(device): - return omnivore.omnivore_swin_b().to(device) - - -def test_omnivore_swin_t_forward(omnivore_swin_t_model, device): - model = omnivore_swin_t_model - - image = torch.randn((1, 3, 1, 112, 112), device=device) # B C D H W - image_score = model(image, input_type="image") - - assert_expected(image_score.size(), torch.Size((1, 1000))) - assert_expected( - image_score.abs().sum(), torch.tensor(184.01417), rtol=1e-3, atol=1e-3 - ) - - rgbd = torch.randn((1, 4, 1, 112, 112), device=device) - rgbd_score = model(rgbd, input_type="rgbd") - assert_expected(rgbd_score.size(), torch.Size((1, 19))) - assert_expected(rgbd_score.abs().sum(), torch.tensor(3.60813), rtol=1e-3, atol=1e-3) - - video = torch.randn((1, 3, 4, 112, 112), device=device) - video_score = model(video, input_type="video") - assert_expected(video_score.size(), torch.Size((1, 400))) - assert_expected( - video_score.abs().sum(), torch.tensor(110.70048), rtol=1e-3, atol=1e-3 - ) - - -def test_omnivore_swin_s_forward(omnivore_swin_s_model, device): - model = omnivore_swin_s_model - - image = torch.randn((1, 3, 1, 112, 112), device=device) # B C D H W - image_score = model(image, input_type="image") - - assert_expected(image_score.size(), torch.Size((1, 1000))) - assert_expected( - image_score.abs().sum(), torch.tensor(239.73104), rtol=1e-3, atol=1e-3 - ) - - rgbd = torch.randn((1, 4, 1, 112, 112), device=device) - rgbd_score = model(rgbd, input_type="rgbd") - assert_expected(rgbd_score.size(), torch.Size((1, 19))) - assert_expected(rgbd_score.abs().sum(), torch.tensor(5.80919), rtol=1e-3, atol=1e-3) - - video = torch.randn((1, 3, 4, 112, 112), device=device) - video_score = model(video, input_type="video") - assert_expected(video_score.size(), torch.Size((1, 400))) - assert_expected( - video_score.abs().sum(), torch.tensor(136.49894), rtol=1e-3, atol=1e-3 - ) - - -def test_omnivore_swin_b_forward(omnivore_swin_b_model, device): - model = omnivore_swin_b_model - - image = torch.randn((1, 3, 1, 112, 112), device=device) # B C D H W - image_score = model(image, input_type="image") - - assert_expected(image_score.size(), torch.Size((1, 1000))) - assert_expected( - image_score.abs().sum(), torch.tensor(278.06488), rtol=1e-3, atol=1e-3 - ) - - rgbd = torch.randn((1, 4, 1, 112, 112), device=device) - rgbd_score = model(rgbd, input_type="rgbd") - assert_expected(rgbd_score.size(), torch.Size((1, 19))) - assert_expected(rgbd_score.abs().sum(), torch.tensor(4.52186), rtol=1e-3, atol=1e-3) - - video = torch.randn((1, 3, 4, 112, 112), device=device) - video_score = model(video, input_type="video") - assert_expected(video_score.size(), torch.Size((1, 400))) - assert_expected( - video_score.abs().sum(), torch.tensor(138.22859), rtol=1e-3, atol=1e-3 - ) - - def test_omnivore_forward_wrong_input_type(omnivore_swin_t_model, device): model = omnivore_swin_t_model