ImagingDataCommons · fedorov · Mar 27, 2026 · Mar 27, 2026 · Mar 27, 2026
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -11,6 +11,19 @@ so only repo-level persistence is reliable.
 Always run `pre-commit run --files <staged files>` before committing to catch
 formatting and linting issues. Fix any failures before creating the commit.
 
+## After modifying SQL queries
+
+Validate every changed SQL file against BigQuery with `--dry_run` before
+committing:
+
+```bash
+bq query --project_id=idc-sandbox-000 --use_legacy_sql=false --dry_run < path/to/query.sql
+```
+
+Fix any query errors before creating the commit. Note that
+`prior_versions_index.sql` uses procedural SQL (`EXECUTE IMMEDIATE`) and cannot
+be validated with `--dry_run`.
+
 ## Before starting any task
 
 Read the developer documentation in `docs/dev/` before exploring the codebase.

diff --git a/assets/ann_group_index.sql b/assets/ann_group_index.sql
@@ -69,3 +69,7 @@ CROSS JOIN
 WHERE
   # Microscopy Bulk Simple Annotations SOP Class UID - more reliable than Modality = "ANN"
   SOPClassUID = "1.2.840.10008.5.1.4.1.1.91.1"
+ORDER BY
+  AnnotationPropertyCategory_CodeMeaning,
+  AnnotationPropertyType_CodeMeaning,
+  AlgorithmName
diff --git a/assets/ann_index.sql b/assets/ann_index.sql
@@ -25,3 +25,5 @@ FROM
 WHERE
   # Microscopy Bulk Simple Annotations SOP Class UID - more reliable than Modality = "ANN"
   SOPClassUID = "1.2.840.10008.5.1.4.1.1.91.1"
+ORDER BY
+  AnnotationCoordinateType, referenced_SeriesInstanceUID
diff --git a/assets/contrast_index.sql b/assets/contrast_index.sql
@@ -8,9 +8,9 @@
 WITH contrast_data AS (
   SELECT
     SeriesInstanceUID,
-    ARRAY_AGG(DISTINCT ContrastBolusAgent IGNORE NULLS) AS ContrastBolusAgent,
-    ARRAY_AGG(DISTINCT ContrastBolusIngredient IGNORE NULLS) AS ContrastBolusIngredient,
-    ARRAY_AGG(DISTINCT ContrastBolusRoute IGNORE NULLS) AS ContrastBolusRoute
+    ARRAY_AGG(DISTINCT ContrastBolusAgent IGNORE NULLS ORDER BY ContrastBolusAgent) AS ContrastBolusAgent,
+    ARRAY_AGG(DISTINCT ContrastBolusIngredient IGNORE NULLS ORDER BY ContrastBolusIngredient) AS ContrastBolusIngredient,
+    ARRAY_AGG(DISTINCT ContrastBolusRoute IGNORE NULLS ORDER BY ContrastBolusRoute) AS ContrastBolusRoute
   FROM `bigquery-public-data.idc_v23.dicom_all`
   WHERE Modality IN ('CT', 'MR', 'PT', 'XA', 'RF')
   GROUP BY SeriesInstanceUID
@@ -37,3 +37,7 @@ WHERE
   ARRAY_LENGTH(ContrastBolusAgent) > 0
   OR ARRAY_LENGTH(ContrastBolusIngredient) > 0
   OR ARRAY_LENGTH(ContrastBolusRoute) > 0
+ORDER BY
+  ContrastBolusAgent[SAFE_OFFSET(0)],
+  ContrastBolusIngredient[SAFE_OFFSET(0)],
+  ContrastBolusRoute[SAFE_OFFSET(0)]
diff --git a/assets/seg_index.sql b/assets/seg_index.sql
@@ -4,6 +4,12 @@
 # key metadata about the segmentation series including
 # the number of segments, segmentation type, algorithm
 # type and name, and the segmented image series.
+# Note: multi-valued columns (AlgorithmType, AlgorithmName,
+# SegmentedPropertyCategory_CodeMeanings, etc.) are aggregated
+# with DISTINCT independently, so positional correspondence
+# between columns is not preserved. For example, the first
+# value in AlgorithmType does not necessarily pair with the
+# first value in AlgorithmName.
 WITH
   segmentations AS (
     WITH
@@ -140,10 +146,10 @@ SELECT
   COUNT(DISTINCT (SegmentNumber)) total_segments,
   # description:
   # Segmentation algorithm type as available in DICOM SegmentAlgorithmType
-  string_agg(DISTINCT (SegmentAlgorithmType), ',') AS AlgorithmType,
+  string_agg(DISTINCT (SegmentAlgorithmType), ',' ORDER BY SegmentAlgorithmType) AS AlgorithmType,
   # description:
   # Segmentation algorithm name as available in DICOM SegmentAlgorithmName
-  string_agg(DISTINCT (SegmentAlgorithmName), ',') AS AlgorithmName,
+  string_agg(DISTINCT (SegmentAlgorithmName), ',' ORDER BY SegmentAlgorithmName) AS AlgorithmName,
   # description:
   # SeriesInstanceUID of the referenced image series that the segmentation applies to
   any_value(segmentations.segmented_SeriesInstanceUID)
@@ -152,20 +158,20 @@ SELECT
   # Array of distinct CodeMeaning values from SegmentedPropertyCategoryCodeSequence across all segments
   # in the series, representing the broad category of the segmented property as defined in DICOM PS3.3 C.8.20.2,
   # e.g., ["Anatomical Structure"], ["Morphologically Altered Structure"]
-  ARRAY_AGG(DISTINCT SegmentedPropertyCategory.CodeMeaning IGNORE NULLS)
+  ARRAY_AGG(DISTINCT SegmentedPropertyCategory.CodeMeaning IGNORE NULLS ORDER BY SegmentedPropertyCategory.CodeMeaning)
     AS SegmentedPropertyCategory_CodeMeanings,
   # description:
   # Array of distinct CodeMeaning values from SegmentedPropertyTypeCodeSequence across all segments
   # in the series, representing the specific type of the segmented property as defined in DICOM PS3.3 C.8.20.2,
   # e.g., ["Liver", "Kidney", "Spleen"]
-  ARRAY_AGG(DISTINCT SegmentedPropertyType.CodeMeaning IGNORE NULLS)
+  ARRAY_AGG(DISTINCT SegmentedPropertyType.CodeMeaning IGNORE NULLS ORDER BY SegmentedPropertyType.CodeMeaning)
     AS SegmentedPropertyType_CodeMeanings,
   # description:
   # Array of distinct CodeMeaning values from AnatomicRegionSequence across all segments
   # in the series, representing the anatomic location of the segmented structure as defined in DICOM PS3.3 C.8.20.2,
   # e.g., ["Abdomen"], ["Thorax", "Head"]
-  ARRAY_AGG(DISTINCT AnatomicRegion.CodeMeaning IGNORE NULLS)
+  ARRAY_AGG(DISTINCT AnatomicRegion.CodeMeaning IGNORE NULLS ORDER BY AnatomicRegion.CodeMeaning)
     AS AnatomicRegion_CodeMeanings
 FROM segmentations
 GROUP BY SeriesInstanceUID
-ORDER BY SegmentationType DESC
+ORDER BY SegmentationType DESC, AlgorithmType, AlgorithmName
diff --git a/assets/sm_index.sql b/assets/sm_index.sql
@@ -21,20 +21,20 @@ WITH
     dicom_all.SeriesInstanceUID,
     ANY_VALUE(ContainerIdentifier) AS ContainerIdentifier,
     ANY_VALUE(Modality) AS Modality,
-    STRING_AGG(DISTINCT(collection_id),",") AS collection_id,
+    ANY_VALUE(collection_id) AS collection_id,
     ANY_VALUE(OpticalPathSequence[SAFE_OFFSET(0)].ObjectiveLensPower) AS ObjectiveLensPower,
     MAX(DISTINCT(TotalPixelMatrixColumns)) AS max_TotalPixelMatrixColumns,
     MAX(DISTINCT(TotalPixelMatrixRows)) AS max_TotalPixelMatrixRows,
     MAX(DISTINCT(`Columns`)) AS max_Columns,
     MAX(DISTINCT(`Rows`)) AS max_Rows,
     MIN(DISTINCT(SAFE_CAST(PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64))) AS min_spacing_0,
     MIN(SAFE_CAST(SharedFunctionalGroupsSequence[SAFE_OFFSET(0)].PixelMeasuresSequence[SAFE_OFFSET(0)]. PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64)) AS fg_min_spacing_0,
-    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS primaryAnatomicStructure_code_str,
-    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS primaryAnatomicStructureModifier_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeMeaning))[SAFE_OFFSET(0)] AS primaryAnatomicStructure_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeMeaning))[SAFE_OFFSET(0)] AS primaryAnatomicStructureModifier_code_str,
 
-    ARRAY_AGG(DISTINCT(CONCAT(OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS illuminationType_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY CONCAT(OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning))[SAFE_OFFSET(0)] AS illuminationType_code_str,
 
-    ARRAY_AGG(DISTINCT(CONCAT(AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS admittingDiagnosis_code_str
+    ARRAY_AGG(DISTINCT(CONCAT(AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY CONCAT(AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeMeaning))[SAFE_OFFSET(0)] AS admittingDiagnosis_code_str
 
 
   FROM
@@ -62,7 +62,7 @@ SpecimenPreparationSequence_unnested AS (
     slide_embedding AS (
     SELECT
       SeriesInstanceUID,
-      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val))) as embeddingMedium_code_str
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) as embeddingMedium_code_str
     FROM SpecimenPreparationSequence_unnested
     WHERE (cnc_csd = 'SCT' and cnc_val = '430863003') -- CodeMeaning is 'Embedding medium'
     GROUP BY SeriesInstanceUID
@@ -71,7 +71,7 @@ SpecimenPreparationSequence_unnested AS (
     slide_fixative AS (
     SELECT
       SeriesInstanceUID,
-      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) as tissueFixative_code_str
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) as tissueFixative_code_str
     FROM SpecimenPreparationSequence_unnested
     WHERE (cnc_csd = 'SCT' and cnc_val = '430864009') -- CodeMeaning is 'Tissue Fixative'
     GROUP BY SeriesInstanceUID
@@ -80,7 +80,7 @@ SpecimenPreparationSequence_unnested AS (
     slide_staining AS (
     SELECT
       SeriesInstanceUID,
-      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) as staining_usingSubstance_code_str,
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) as staining_usingSubstance_code_str,
     FROM SpecimenPreparationSequence_unnested
     WHERE (cnc_csd = 'SCT' and cnc_val = '424361007') -- CodeMeaning is 'Using substance'
     GROUP BY SeriesInstanceUID
@@ -183,3 +183,7 @@ LEFT JOIN slide_fixative on temp_table.SeriesInstanceUID = slide_fixative.Series
 LEFT JOIN slide_staining on temp_table.SeriesInstanceUID = slide_staining.SeriesInstanceUID
 WHERE
   Modality = "SM"
+ORDER BY
+  primaryAnatomicStructure_CodeMeaning,
+  staining_usingSubstance_CodeMeaning[SAFE_OFFSET(0)],
+  collection_id
diff --git a/assets/sm_instance_index.sql b/assets/sm_instance_index.sql
@@ -24,7 +24,7 @@ WITH
   slide_embedding AS (
   SELECT
     SOPInstanceUID,
-    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val))) AS embeddingMedium_code_str
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) AS embeddingMedium_code_str
   FROM
     SpecimenPreparationSequence_unnested
   WHERE
@@ -35,7 +35,7 @@ WITH
   slide_fixative AS (
   SELECT
     SOPInstanceUID,
-    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) AS tissueFixative_code_str
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) AS tissueFixative_code_str
   FROM
     SpecimenPreparationSequence_unnested
   WHERE
@@ -46,7 +46,7 @@ WITH
   slide_staining AS (
   SELECT
     SOPInstanceUID,
-    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) AS staining_usingSubstance_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) AS staining_usingSubstance_code_str,
   FROM
     SpecimenPreparationSequence_unnested
   WHERE
@@ -160,4 +160,6 @@ ON
 WHERE
   dicom_all.Modality="SM"
 ORDER BY
-  SeriesInstanceUID DESC
+  staining_usingSubstance_CodeMeaning[SAFE_OFFSET(0)],
+  TransferSyntaxUID,
+  SeriesInstanceUID
diff --git a/scripts/gdc/idc_gdc_selection.sql b/scripts/gdc/idc_gdc_selection.sql
@@ -6,7 +6,7 @@ WITH
       StudyInstanceUID,
       ANY_VALUE(StudyDate) AS StudyDate,
       ANY_VALUE(StudyDescription) AS StudyDescription,
-      ARRAY_AGG(DISTINCT Modality) AS Modalities
+      ARRAY_AGG(DISTINCT Modality ORDER BY Modality) AS Modalities
     FROM
       `bigquery-public-data.idc_current.dicom_all`
     WHERE

diff --git a/scripts/sql/analysis_results_index.sql b/scripts/sql/analysis_results_index.sql
@@ -44,3 +44,5 @@ SELECT
   Citation
 FROM
   `bigquery-public-data.idc_v23.analysis_results_metadata`
+ORDER BY
+  analysis_result_id
diff --git a/scripts/sql/collections_index.sql b/scripts/sql/collections_index.sql
@@ -41,3 +41,5 @@ SELECT
   Description
 FROM
   `bigquery-public-data.idc_v23.original_collections_metadata`
+ORDER BY
+  collection_id
diff --git a/scripts/sql/idc_index.sql b/scripts/sql/idc_index.sql
@@ -170,3 +170,5 @@ ON
   dicom_all.SOPInstanceUID = dicom_curated.SOPInstanceUID
 GROUP BY
   SeriesInstanceUID
+ORDER BY
+  collection_id, PatientID, StudyInstanceUID, SeriesInstanceUID
diff --git a/scripts/sql/prior_versions_index.sql b/scripts/sql/prior_versions_index.sql
@@ -45,6 +45,7 @@ SET union_all_query = (
   """,
   version, version, latest_idc_version),
     " UNION ALL "
+    ORDER BY version
   )
   FROM UNNEST(idc_versions) AS version
 );
@@ -95,6 +96,8 @@ where gcs_bucket not in ('idc-open-idc')
 
 GROUP BY
  1,2,3,4,5,6,7,8
+ORDER BY
+ collection_id, min_idc_version, Modality
   """,
   union_all_query
 );