enh: improve schema column descriptions

* require that descriptions are not empty for all queries parsed
* add missing descriptions
* add explicit short column names whenever source column was dereferenced
from a table (e.g., "dicom_all.ImageType AS ImageType")
* fix incorrect baseline

Author: fedorov

assets/sm_instance_index.sql

@@ -50,10 +50,10 @@ WITH
 SELECT
   # description:
   # unique identifier of the instance
-  dicom_all.SOPInstanceUID,
+  dicom_all.SOPInstanceUID AS SOPInstanceUID,
   # description:
   # unique identifier of the series
-  dicom_all.SeriesInstanceUID,
+  dicom_all.SeriesInstanceUID AS SeriesInstanceUID,
   -- Embedding Medium
   # description:
   # embedding medium used for the slide preparation
@@ -119,23 +119,23 @@ SELECT
   SAFE_CAST(SharedFunctionalGroupsSequence[SAFE_OFFSET(0)].PixelMeasuresSequence[SAFE_OFFSET(0)]. PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64) AS PixelSpacing_0,
   # description:
   # DICOM ImageType attribute
-  dicom_all.ImageType,
+  dicom_all.ImageType AS ImageType,
   # description:
   # DICOM TransferSyntaxUID attribute
-  dicom_all.TransferSyntaxUID,
+  dicom_all.TransferSyntaxUID AS TransferSyntaxUID,
   # description:
   # size of the instance file in bytes
-  dicom_all.instance_size,
+  dicom_all.instance_size AS instance_size,
   # description:
   # number of columns in the image
-  dicom_all.TotalPixelMatrixColumns,
+  dicom_all.TotalPixelMatrixColumns AS TotalPixelMatrixColumns,
   # description:
   # number of rows in the image
-  dicom_all.TotalPixelMatrixRows,
+  dicom_all.TotalPixelMatrixRows AS TotalPixelMatrixRows,
   -- attributes needed to retrieve the selected instances/files
   # description:
   # unique identifier of the instance within the IDC
-  dicom_all.crdc_instance_uuid
+  dicom_all.crdc_instance_uuid AS crdc_instance_uuid
 FROM
   `bigquery-public-data.idc_v22.dicom_all` AS dicom_all
 LEFT JOIN

pyproject.toml

@@ -13,7 +13,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "idc-index-data"
-version = "22.1.3"
+version = "22.1.4"
 authors = [
   { name = "Andrey Fedorov", email = "andrey.fedorov@gmail.com" },
   { name = "Vamsi Thiriveedhi", email = "vthiriveedhi@mgh.harvard.edu" },

scripts/python/idc_index_data_manager.py

@@ -130,10 +130,16 @@ def parse_column_descriptions(sql_query: str) -> dict[str, str]:
                         logger.debug(
                             "Parsed description for column '%s': %s",
                             column_name,
-                            description[:50] + "..."
-                            if len(description) > 50
-                            else description,
+                            description,
                         )
+                        # throw exception if description is empty
+                        if not description:
+                            raise ValueError(
+                                "Description for column '"
+                                + column_name
+                                + "' is empty, and empty descriptions are not allowed."
+                            )
+
                 else:
                     i += 1
             else:

scripts/sql/analysis_results_index.sql

@@ -1,16 +1,42 @@
 SELECT
+  # description:
+  # unique identifier of the analysis results collection
   ID AS analysis_result_id,
+  # description:
+  # name of the analysis results collection
   Title AS analysis_result_title,
+  # description:
+  # Digital Object Identifier (DOI) of the analysis results collection
   source_doi,
+  # description:
+  # URL for the location of additional information about the analysis results collection
   source_url,
+  # description:
+  # number of subjects analyzed in the analysis results collection
   Subjects,
+  # description:
+  # collections analyzed in the analysis results collection
   Collections,
+  # description:
+  # analysis artifacts included in the analysis results collection
   AnalysisArtifacts,
+  # description:
+  # timestamp of the last update to the analysis results collection
   Updated,
+  # description:
+  # license URL for the analysis results collection
   license_url,
+  # description:
+  # license name for the analysis results collection
   license_long_name,
+  # description:
+  # short name for the license of the analysis results collection
   license_short_name,
+  # description:
+  # detailed description of the analysis results collection
   Description,
+  # description:
+  # citation for the analysis results collection that should be used for acknowledgment
   Citation
 FROM
   `bigquery-public-data.idc_v22.analysis_results_metadata`

tests/test_real_sql_parsing.py

@@ -84,18 +84,6 @@ def test_real_sql_files() -> None:
             else:
                 print(f"✗ Missing expected column: {col}")
 
-    # Test analysis_results_index.sql (should have no descriptions)
-    analysis_sql_path = sql_dir / "analysis_results_index.sql"
-    if analysis_sql_path.exists():
-        with analysis_sql_path.open("r") as f:
-            sql_query = f.read()
-
-        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
-        print("\n=== analysis_results_index.sql ===")
-        print(f"Found {len(descriptions)} column descriptions (expected 0)")
-        assert len(descriptions) == 0, "Expected no descriptions in this file"
-        print("✓ Correctly found no descriptions")
-
 
 if __name__ == "__main__":
     test_real_sql_files()