Comments addressing PR #15 and Issue #65

Author: russellhowes

configs/train/vitg16/droid-256px-8f.yaml

@@ -10,7 +10,9 @@ data:
   - left_mp4_path
   crop_size: 256
   datasets:
-    - /your_file_path/droid_train_paths_cw.csv
+    # CSV file, each line is a trajectory dir path
+    # See https://github.com/facebookresearch/vjepa2/issues/65
+    - /your_file_path/droid_train_paths.csv
   dataset_fpcs:
   - 8
   fps: 4

src/models/utils/modules.py

@@ -36,9 +36,13 @@ def rotate_queries_or_keys(x, pos):
     # -- build rotation matrix and apply
     emb_sin = freq.sin()  # (..., N, D/2)
     emb_cos = freq.cos()  # (..., N, D/2)
-
+    # -- NOTE: This expansion has a subtle bug where frequencies are duplicated across the vector pair.
+    # -- Fixing the bug would break compatibility with the pretrained model, but the fix can be applied by commenting
+    # -- out the two lines below, and uncommenting the following two lines. (Thanks to @echosprint)
     emb_sin = emb_sin.squeeze(-1).repeat(1, 1, 1, 2)
     emb_cos = emb_cos.squeeze(-1).repeat(1, 1, 1, 2)
+    # emb_sin = emb_sin.repeat_interleave(2, dim=-1)  # (..., N, D)
+    # emb_cos = emb_cos.repeat_interleave(2, dim=-1)  # (..., N, D)
 
     # --
     y = x.unflatten(-1, (-1, 2))