Add description.

Author: mihaidusmanu

README.md

@@ -189,8 +189,21 @@ This executes two steps:
 
 :three: __Obtain the evaluation results:__
 
-- validation queries: the script print the localization recall.
-- test queries: combine results for all locations / devices in a single zip file using `python -m lamar.combine_results --cab_phone_path poses_CAB_phone.txt [...] --output_dir ./result/` and submit the zip to the [benchmark page](https://www.codabench.org/competitions/7918/).
+_Validation queries:_ the script print the localization recall.
+
+_Test queries:_
+- Combine results for all locations / devices in a single zip file using
+
+```
+python -m lamar.combine_results \
+  --cab_phone_path path/to/poses_CAB_phone.txt [...] \
+  --description path/to/description.txt \
+  --output_dir ./result/
+```
+
+- Submit the zip to the [benchmark page](https://www.codabench.org/competitions/7918/).
+
+:warning: Any public entries should have a name and valid link on the leaderboard. The description file should contain details about the submission (i.e, what methods, what thresholds / hyperparameters, ...). 
 
 :four: __Workflow:__ the benchmarking pipeline is designed such that
 - the mapping and localization process is split into modular steps listed in [`lamar/tasks/`](./lamar/tasks/)

lamar/combine_results.py

@@ -5,6 +5,13 @@
 
 from . import logger
 
+def assert_valid_txt_path(path: Path):
+    """Assert that the path is valid."""
+    assert path.exists(), f"Path does not exist: {path}"
+    assert path.is_file(), f"Path is not a file: {path}"
+    assert path.suffix == ".txt", f"Expected .txt file, got {path.suffix}"
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
         description="Combine estimated poses from multiple scenes / devices in a zip file for evaluation."
@@ -45,14 +52,21 @@
         default=None,
         help="File containing the HGE HoloLens estimated poses.",
     )
+    parser.add_argument(
+        "--description_path",
+        type=Path,
+        required=True,
+        help="Path to a text file containing description of the submission.",
+    )
     parser.add_argument(
         "--output_dir",
         type=Path,
-        help="Output directory where the zip will be saved.",
         required=True,
+        help="Output directory where the zip will be saved.",
     )
     args = parser.parse_args().__dict__
     output_dir = args.pop("output_dir")
+    description_path = args.pop("description_path")
 
     # Generate timestamp for the zip file name.
     timestamp = datetime.datetime.now().strftime("%Y.%m.%d_%H.%M.%S")
@@ -62,6 +76,10 @@
     logger.info(f"Creating zip file at {zip_filename}")
     output_dir.mkdir(parents=True, exist_ok=True)
     with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf:
+        # Add the description file to the zip.
+        logger.info(f"Adding description file from {description_path} to zip")
+        assert_valid_txt_path(description_path)
+        zipf.write(description_path, arcname="description.txt")
         for name, path in args.items():
             split = name.split("_")
             assert len(split) == 3
@@ -70,9 +88,7 @@
             if path is None:
                 logger.warning(f"No path provided for [{split}], skipping...")
                 continue
-            assert path.exists()
-            assert path.is_file()
-            assert path.suffix == ".txt", f"Expected .txt file, got {path.suffix}"
             logger.info(f"Adding [{split}] file from {path} to zip")
+            assert_valid_txt_path(path)
             zipf.write(path, arcname=f"{split[0].upper()}_query_{split[1]}.txt")
     logger.info(f"Successfully created zip file at {zip_filename}")