From a3c441a1761440ee1ad0b2e9b202182e634dbbeb Mon Sep 17 00:00:00 2001
From: Andrey Fedorov <andrey.fedorov@gmail.com>
Date: Fri, 27 Mar 2026 16:17:14 -0400
Subject: [PATCH 1/2] ENH: add deterministic ordering to aggregations and
 output for better Parquet compression

Add ORDER BY to all ARRAY_AGG/STRING_AGG calls that lacked it, ensuring
identical logical arrays are always encoded the same way (improving
dictionary encoding). Add compression-friendly final ORDER BY to all
queries, clustering rows by semantically meaningful columns (anatomy,
staining, collection) rather than arbitrary UIDs to maximize run-length
and dictionary encoding in the output Parquet files.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 assets/ann_group_index.sql             |  4 ++++
 assets/ann_index.sql                   |  2 ++
 assets/contrast_index.sql              | 10 +++++++---
 assets/seg_index.sql                   | 18 ++++++++++++------
 assets/sm_index.sql                    | 20 ++++++++++++--------
 assets/sm_instance_index.sql           | 10 ++++++----
 scripts/gdc/idc_gdc_selection.sql      |  2 +-
 scripts/sql/analysis_results_index.sql |  2 ++
 scripts/sql/collections_index.sql      |  2 ++
 scripts/sql/idc_index.sql              |  2 ++
 scripts/sql/prior_versions_index.sql   |  3 +++
 11 files changed, 53 insertions(+), 22 deletions(-)

diff --git a/assets/ann_group_index.sql b/assets/ann_group_index.sql
index 1b5e8ca..5410f23 100644
--- a/assets/ann_group_index.sql
+++ b/assets/ann_group_index.sql
@@ -69,3 +69,7 @@ CROSS JOIN
 WHERE
   # Microscopy Bulk Simple Annotations SOP Class UID - more reliable than Modality = "ANN"
   SOPClassUID = "1.2.840.10008.5.1.4.1.1.91.1"
+ORDER BY
+  AnnotationPropertyCategory_CodeMeaning,
+  AnnotationPropertyType_CodeMeaning,
+  AlgorithmName
diff --git a/assets/ann_index.sql b/assets/ann_index.sql
index 9b6dda2..13a4a31 100644
--- a/assets/ann_index.sql
+++ b/assets/ann_index.sql
@@ -25,3 +25,5 @@ FROM
 WHERE
   # Microscopy Bulk Simple Annotations SOP Class UID - more reliable than Modality = "ANN"
   SOPClassUID = "1.2.840.10008.5.1.4.1.1.91.1"
+ORDER BY
+  AnnotationCoordinateType, referenced_SeriesInstanceUID
diff --git a/assets/contrast_index.sql b/assets/contrast_index.sql
index 6e95f0b..c920192 100644
--- a/assets/contrast_index.sql
+++ b/assets/contrast_index.sql
@@ -8,9 +8,9 @@
 WITH contrast_data AS (
   SELECT
     SeriesInstanceUID,
-    ARRAY_AGG(DISTINCT ContrastBolusAgent IGNORE NULLS) AS ContrastBolusAgent,
-    ARRAY_AGG(DISTINCT ContrastBolusIngredient IGNORE NULLS) AS ContrastBolusIngredient,
-    ARRAY_AGG(DISTINCT ContrastBolusRoute IGNORE NULLS) AS ContrastBolusRoute
+    ARRAY_AGG(DISTINCT ContrastBolusAgent IGNORE NULLS ORDER BY 1) AS ContrastBolusAgent,
+    ARRAY_AGG(DISTINCT ContrastBolusIngredient IGNORE NULLS ORDER BY 1) AS ContrastBolusIngredient,
+    ARRAY_AGG(DISTINCT ContrastBolusRoute IGNORE NULLS ORDER BY 1) AS ContrastBolusRoute
   FROM `bigquery-public-data.idc_v23.dicom_all`
   WHERE Modality IN ('CT', 'MR', 'PT', 'XA', 'RF')
   GROUP BY SeriesInstanceUID
@@ -37,3 +37,7 @@ WHERE
   ARRAY_LENGTH(ContrastBolusAgent) > 0
   OR ARRAY_LENGTH(ContrastBolusIngredient) > 0
   OR ARRAY_LENGTH(ContrastBolusRoute) > 0
+ORDER BY
+  ContrastBolusAgent,
+  ContrastBolusIngredient,
+  ContrastBolusRoute
diff --git a/assets/seg_index.sql b/assets/seg_index.sql
index 69de09a..6130e1d 100644
--- a/assets/seg_index.sql
+++ b/assets/seg_index.sql
@@ -4,6 +4,12 @@
 # key metadata about the segmentation series including
 # the number of segments, segmentation type, algorithm
 # type and name, and the segmented image series.
+# Note: multi-valued columns (AlgorithmType, AlgorithmName,
+# SegmentedPropertyCategory_CodeMeanings, etc.) are aggregated
+# with DISTINCT independently, so positional correspondence
+# between columns is not preserved. For example, the first
+# value in AlgorithmType does not necessarily pair with the
+# first value in AlgorithmName.
 WITH
   segmentations AS (
     WITH
@@ -140,10 +146,10 @@ SELECT
   COUNT(DISTINCT (SegmentNumber)) total_segments,
   # description:
   # Segmentation algorithm type as available in DICOM SegmentAlgorithmType
-  string_agg(DISTINCT (SegmentAlgorithmType), ',') AS AlgorithmType,
+  string_agg(DISTINCT (SegmentAlgorithmType), ',' ORDER BY 1) AS AlgorithmType,
   # description:
   # Segmentation algorithm name as available in DICOM SegmentAlgorithmName
-  string_agg(DISTINCT (SegmentAlgorithmName), ',') AS AlgorithmName,
+  string_agg(DISTINCT (SegmentAlgorithmName), ',' ORDER BY 1) AS AlgorithmName,
   # description:
   # SeriesInstanceUID of the referenced image series that the segmentation applies to
   any_value(segmentations.segmented_SeriesInstanceUID)
@@ -152,20 +158,20 @@ SELECT
   # Array of distinct CodeMeaning values from SegmentedPropertyCategoryCodeSequence across all segments
   # in the series, representing the broad category of the segmented property as defined in DICOM PS3.3 C.8.20.2,
   # e.g., ["Anatomical Structure"], ["Morphologically Altered Structure"]
-  ARRAY_AGG(DISTINCT SegmentedPropertyCategory.CodeMeaning IGNORE NULLS)
+  ARRAY_AGG(DISTINCT SegmentedPropertyCategory.CodeMeaning IGNORE NULLS ORDER BY 1)
     AS SegmentedPropertyCategory_CodeMeanings,
   # description:
   # Array of distinct CodeMeaning values from SegmentedPropertyTypeCodeSequence across all segments
   # in the series, representing the specific type of the segmented property as defined in DICOM PS3.3 C.8.20.2,
   # e.g., ["Liver", "Kidney", "Spleen"]
-  ARRAY_AGG(DISTINCT SegmentedPropertyType.CodeMeaning IGNORE NULLS)
+  ARRAY_AGG(DISTINCT SegmentedPropertyType.CodeMeaning IGNORE NULLS ORDER BY 1)
     AS SegmentedPropertyType_CodeMeanings,
   # description:
   # Array of distinct CodeMeaning values from AnatomicRegionSequence across all segments
   # in the series, representing the anatomic location of the segmented structure as defined in DICOM PS3.3 C.8.20.2,
   # e.g., ["Abdomen"], ["Thorax", "Head"]
-  ARRAY_AGG(DISTINCT AnatomicRegion.CodeMeaning IGNORE NULLS)
+  ARRAY_AGG(DISTINCT AnatomicRegion.CodeMeaning IGNORE NULLS ORDER BY 1)
     AS AnatomicRegion_CodeMeanings
 FROM segmentations
 GROUP BY SeriesInstanceUID
-ORDER BY SegmentationType DESC
+ORDER BY SegmentationType DESC, AlgorithmType, AlgorithmName
diff --git a/assets/sm_index.sql b/assets/sm_index.sql
index 58e9f29..8edb373 100644
--- a/assets/sm_index.sql
+++ b/assets/sm_index.sql
@@ -21,7 +21,7 @@ WITH
     dicom_all.SeriesInstanceUID,
     ANY_VALUE(ContainerIdentifier) AS ContainerIdentifier,
     ANY_VALUE(Modality) AS Modality,
-    STRING_AGG(DISTINCT(collection_id),",") AS collection_id,
+    ANY_VALUE(collection_id) AS collection_id,
     ANY_VALUE(OpticalPathSequence[SAFE_OFFSET(0)].ObjectiveLensPower) AS ObjectiveLensPower,
     MAX(DISTINCT(TotalPixelMatrixColumns)) AS max_TotalPixelMatrixColumns,
     MAX(DISTINCT(TotalPixelMatrixRows)) AS max_TotalPixelMatrixRows,
@@ -29,12 +29,12 @@ WITH
     MAX(DISTINCT(`Rows`)) AS max_Rows,
     MIN(DISTINCT(SAFE_CAST(PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64))) AS min_spacing_0,
     MIN(SAFE_CAST(SharedFunctionalGroupsSequence[SAFE_OFFSET(0)].PixelMeasuresSequence[SAFE_OFFSET(0)]. PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64)) AS fg_min_spacing_0,
-    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS primaryAnatomicStructure_code_str,
-    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS primaryAnatomicStructureModifier_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY 1)[SAFE_OFFSET(0)] AS primaryAnatomicStructure_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY 1)[SAFE_OFFSET(0)] AS primaryAnatomicStructureModifier_code_str,
 
-    ARRAY_AGG(DISTINCT(CONCAT(OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS illuminationType_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY 1)[SAFE_OFFSET(0)] AS illuminationType_code_str,
 
-    ARRAY_AGG(DISTINCT(CONCAT(AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS admittingDiagnosis_code_str
+    ARRAY_AGG(DISTINCT(CONCAT(AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY 1)[SAFE_OFFSET(0)] AS admittingDiagnosis_code_str
 
 
   FROM
@@ -62,7 +62,7 @@ SpecimenPreparationSequence_unnested AS (
     slide_embedding AS (
     SELECT
       SeriesInstanceUID,
-      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val))) as embeddingMedium_code_str
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) ORDER BY 1) as embeddingMedium_code_str
     FROM SpecimenPreparationSequence_unnested
     WHERE (cnc_csd = 'SCT' and cnc_val = '430863003') -- CodeMeaning is 'Embedding medium'
     GROUP BY SeriesInstanceUID
@@ -71,7 +71,7 @@ SpecimenPreparationSequence_unnested AS (
     slide_fixative AS (
     SELECT
       SeriesInstanceUID,
-      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) as tissueFixative_code_str
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY 1) as tissueFixative_code_str
     FROM SpecimenPreparationSequence_unnested
     WHERE (cnc_csd = 'SCT' and cnc_val = '430864009') -- CodeMeaning is 'Tissue Fixative'
     GROUP BY SeriesInstanceUID
@@ -80,7 +80,7 @@ SpecimenPreparationSequence_unnested AS (
     slide_staining AS (
     SELECT
       SeriesInstanceUID,
-      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) as staining_usingSubstance_code_str,
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY 1) as staining_usingSubstance_code_str,
     FROM SpecimenPreparationSequence_unnested
     WHERE (cnc_csd = 'SCT' and cnc_val = '424361007') -- CodeMeaning is 'Using substance'
     GROUP BY SeriesInstanceUID
@@ -183,3 +183,7 @@ LEFT JOIN slide_fixative on temp_table.SeriesInstanceUID = slide_fixative.Series
 LEFT JOIN slide_staining on temp_table.SeriesInstanceUID = slide_staining.SeriesInstanceUID
 WHERE
   Modality = "SM"
+ORDER BY
+  primaryAnatomicStructure_CodeMeaning,
+  staining_usingSubstance_CodeMeaning,
+  collection_id
diff --git a/assets/sm_instance_index.sql b/assets/sm_instance_index.sql
index 248a5a0..f116a6a 100644
--- a/assets/sm_instance_index.sql
+++ b/assets/sm_instance_index.sql
@@ -24,7 +24,7 @@ WITH
   slide_embedding AS (
   SELECT
     SOPInstanceUID,
-    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val))) AS embeddingMedium_code_str
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) ORDER BY 1) AS embeddingMedium_code_str
   FROM
     SpecimenPreparationSequence_unnested
   WHERE
@@ -35,7 +35,7 @@ WITH
   slide_fixative AS (
   SELECT
     SOPInstanceUID,
-    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) AS tissueFixative_code_str
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY 1) AS tissueFixative_code_str
   FROM
     SpecimenPreparationSequence_unnested
   WHERE
@@ -46,7 +46,7 @@ WITH
   slide_staining AS (
   SELECT
     SOPInstanceUID,
-    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) AS staining_usingSubstance_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY 1) AS staining_usingSubstance_code_str,
   FROM
     SpecimenPreparationSequence_unnested
   WHERE
@@ -160,4 +160,6 @@ ON
 WHERE
   dicom_all.Modality="SM"
 ORDER BY
-  SeriesInstanceUID DESC
+  staining_usingSubstance_CodeMeaning,
+  TransferSyntaxUID,
+  SeriesInstanceUID
diff --git a/scripts/gdc/idc_gdc_selection.sql b/scripts/gdc/idc_gdc_selection.sql
index c25fcb9..a23c844 100644
--- a/scripts/gdc/idc_gdc_selection.sql
+++ b/scripts/gdc/idc_gdc_selection.sql
@@ -6,7 +6,7 @@ WITH
       StudyInstanceUID,
       ANY_VALUE(StudyDate) AS StudyDate,
       ANY_VALUE(StudyDescription) AS StudyDescription,
-      ARRAY_AGG(DISTINCT Modality) AS Modalities
+      ARRAY_AGG(DISTINCT Modality ORDER BY 1) AS Modalities
     FROM
       `bigquery-public-data.idc_current.dicom_all`
     WHERE
diff --git a/scripts/sql/analysis_results_index.sql b/scripts/sql/analysis_results_index.sql
index 7a1fe5e..bf36b25 100644
--- a/scripts/sql/analysis_results_index.sql
+++ b/scripts/sql/analysis_results_index.sql
@@ -44,3 +44,5 @@ SELECT
   Citation
 FROM
   `bigquery-public-data.idc_v23.analysis_results_metadata`
+ORDER BY
+  analysis_result_id
diff --git a/scripts/sql/collections_index.sql b/scripts/sql/collections_index.sql
index 0b91021..6ac75ee 100644
--- a/scripts/sql/collections_index.sql
+++ b/scripts/sql/collections_index.sql
@@ -41,3 +41,5 @@ SELECT
   Description
 FROM
   `bigquery-public-data.idc_v23.original_collections_metadata`
+ORDER BY
+  collection_id
diff --git a/scripts/sql/idc_index.sql b/scripts/sql/idc_index.sql
index 856624a..e51ca3f 100644
--- a/scripts/sql/idc_index.sql
+++ b/scripts/sql/idc_index.sql
@@ -170,3 +170,5 @@ ON
   dicom_all.SOPInstanceUID = dicom_curated.SOPInstanceUID
 GROUP BY
   SeriesInstanceUID
+ORDER BY
+  collection_id, PatientID, StudyInstanceUID, SeriesInstanceUID
diff --git a/scripts/sql/prior_versions_index.sql b/scripts/sql/prior_versions_index.sql
index b41f929..563fa99 100644
--- a/scripts/sql/prior_versions_index.sql
+++ b/scripts/sql/prior_versions_index.sql
@@ -45,6 +45,7 @@ SET union_all_query = (
   """,
   version, version, latest_idc_version),
     " UNION ALL "
+    ORDER BY version
   )
   FROM UNNEST(idc_versions) AS version
 );
@@ -95,6 +96,8 @@ where gcs_bucket not in ('idc-open-idc')
 
 GROUP BY
  1,2,3,4,5,6,7,8
+ORDER BY
+ collection_id, min_idc_version, Modality
   """,
   union_all_query
 );

From f422be8ff879ea8637bf077317bc3bcaf05dd149 Mon Sep 17 00:00:00 2001
From: Andrey Fedorov <andrey.fedorov@gmail.com>
Date: Fri, 27 Mar 2026 16:33:37 -0400
Subject: [PATCH 2/2] FIX: replace ORDER BY 1 with explicit expressions for
 BigQuery compatibility

BigQuery does not support positional ORDER BY inside aggregate functions.
Replace all ORDER BY 1 with the actual aggregated expression. Also fix
final ORDER BY clauses that referenced ARRAY columns (not sortable in BQ)
by using [SAFE_OFFSET(0)] to extract the first element. Add BQ dry-run
validation guidance to CLAUDE.md.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 CLAUDE.md                         | 13 +++++++++++++
 assets/contrast_index.sql         | 12 ++++++------
 assets/seg_index.sql              | 10 +++++-----
 assets/sm_index.sql               | 16 ++++++++--------
 assets/sm_instance_index.sql      |  8 ++++----
 scripts/gdc/idc_gdc_selection.sql |  2 +-
 6 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index 5113a58..0ee0d58 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -11,6 +11,19 @@ so only repo-level persistence is reliable.
 Always run `pre-commit run --files <staged files>` before committing to catch
 formatting and linting issues. Fix any failures before creating the commit.
 
+## After modifying SQL queries
+
+Validate every changed SQL file against BigQuery with `--dry_run` before
+committing:
+
+```bash
+bq query --project_id=idc-sandbox-000 --use_legacy_sql=false --dry_run < path/to/query.sql
+```
+
+Fix any query errors before creating the commit. Note that
+`prior_versions_index.sql` uses procedural SQL (`EXECUTE IMMEDIATE`) and cannot
+be validated with `--dry_run`.
+
 ## Before starting any task
 
 Read the developer documentation in `docs/dev/` before exploring the codebase.
diff --git a/assets/contrast_index.sql b/assets/contrast_index.sql
index c920192..7174ccc 100644
--- a/assets/contrast_index.sql
+++ b/assets/contrast_index.sql
@@ -8,9 +8,9 @@
 WITH contrast_data AS (
   SELECT
     SeriesInstanceUID,
-    ARRAY_AGG(DISTINCT ContrastBolusAgent IGNORE NULLS ORDER BY 1) AS ContrastBolusAgent,
-    ARRAY_AGG(DISTINCT ContrastBolusIngredient IGNORE NULLS ORDER BY 1) AS ContrastBolusIngredient,
-    ARRAY_AGG(DISTINCT ContrastBolusRoute IGNORE NULLS ORDER BY 1) AS ContrastBolusRoute
+    ARRAY_AGG(DISTINCT ContrastBolusAgent IGNORE NULLS ORDER BY ContrastBolusAgent) AS ContrastBolusAgent,
+    ARRAY_AGG(DISTINCT ContrastBolusIngredient IGNORE NULLS ORDER BY ContrastBolusIngredient) AS ContrastBolusIngredient,
+    ARRAY_AGG(DISTINCT ContrastBolusRoute IGNORE NULLS ORDER BY ContrastBolusRoute) AS ContrastBolusRoute
   FROM `bigquery-public-data.idc_v23.dicom_all`
   WHERE Modality IN ('CT', 'MR', 'PT', 'XA', 'RF')
   GROUP BY SeriesInstanceUID
@@ -38,6 +38,6 @@ WHERE
   OR ARRAY_LENGTH(ContrastBolusIngredient) > 0
   OR ARRAY_LENGTH(ContrastBolusRoute) > 0
 ORDER BY
-  ContrastBolusAgent,
-  ContrastBolusIngredient,
-  ContrastBolusRoute
+  ContrastBolusAgent[SAFE_OFFSET(0)],
+  ContrastBolusIngredient[SAFE_OFFSET(0)],
+  ContrastBolusRoute[SAFE_OFFSET(0)]
diff --git a/assets/seg_index.sql b/assets/seg_index.sql
index 6130e1d..f820f7a 100644
--- a/assets/seg_index.sql
+++ b/assets/seg_index.sql
@@ -146,10 +146,10 @@ SELECT
   COUNT(DISTINCT (SegmentNumber)) total_segments,
   # description:
   # Segmentation algorithm type as available in DICOM SegmentAlgorithmType
-  string_agg(DISTINCT (SegmentAlgorithmType), ',' ORDER BY 1) AS AlgorithmType,
+  string_agg(DISTINCT (SegmentAlgorithmType), ',' ORDER BY SegmentAlgorithmType) AS AlgorithmType,
   # description:
   # Segmentation algorithm name as available in DICOM SegmentAlgorithmName
-  string_agg(DISTINCT (SegmentAlgorithmName), ',' ORDER BY 1) AS AlgorithmName,
+  string_agg(DISTINCT (SegmentAlgorithmName), ',' ORDER BY SegmentAlgorithmName) AS AlgorithmName,
   # description:
   # SeriesInstanceUID of the referenced image series that the segmentation applies to
   any_value(segmentations.segmented_SeriesInstanceUID)
@@ -158,19 +158,19 @@ SELECT
   # Array of distinct CodeMeaning values from SegmentedPropertyCategoryCodeSequence across all segments
   # in the series, representing the broad category of the segmented property as defined in DICOM PS3.3 C.8.20.2,
   # e.g., ["Anatomical Structure"], ["Morphologically Altered Structure"]
-  ARRAY_AGG(DISTINCT SegmentedPropertyCategory.CodeMeaning IGNORE NULLS ORDER BY 1)
+  ARRAY_AGG(DISTINCT SegmentedPropertyCategory.CodeMeaning IGNORE NULLS ORDER BY SegmentedPropertyCategory.CodeMeaning)
     AS SegmentedPropertyCategory_CodeMeanings,
   # description:
   # Array of distinct CodeMeaning values from SegmentedPropertyTypeCodeSequence across all segments
   # in the series, representing the specific type of the segmented property as defined in DICOM PS3.3 C.8.20.2,
   # e.g., ["Liver", "Kidney", "Spleen"]
-  ARRAY_AGG(DISTINCT SegmentedPropertyType.CodeMeaning IGNORE NULLS ORDER BY 1)
+  ARRAY_AGG(DISTINCT SegmentedPropertyType.CodeMeaning IGNORE NULLS ORDER BY SegmentedPropertyType.CodeMeaning)
     AS SegmentedPropertyType_CodeMeanings,
   # description:
   # Array of distinct CodeMeaning values from AnatomicRegionSequence across all segments
   # in the series, representing the anatomic location of the segmented structure as defined in DICOM PS3.3 C.8.20.2,
   # e.g., ["Abdomen"], ["Thorax", "Head"]
-  ARRAY_AGG(DISTINCT AnatomicRegion.CodeMeaning IGNORE NULLS ORDER BY 1)
+  ARRAY_AGG(DISTINCT AnatomicRegion.CodeMeaning IGNORE NULLS ORDER BY AnatomicRegion.CodeMeaning)
     AS AnatomicRegion_CodeMeanings
 FROM segmentations
 GROUP BY SeriesInstanceUID
diff --git a/assets/sm_index.sql b/assets/sm_index.sql
index 8edb373..c2ce66e 100644
--- a/assets/sm_index.sql
+++ b/assets/sm_index.sql
@@ -29,12 +29,12 @@ WITH
     MAX(DISTINCT(`Rows`)) AS max_Rows,
     MIN(DISTINCT(SAFE_CAST(PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64))) AS min_spacing_0,
     MIN(SAFE_CAST(SharedFunctionalGroupsSequence[SAFE_OFFSET(0)].PixelMeasuresSequence[SAFE_OFFSET(0)]. PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64)) AS fg_min_spacing_0,
-    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY 1)[SAFE_OFFSET(0)] AS primaryAnatomicStructure_code_str,
-    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY 1)[SAFE_OFFSET(0)] AS primaryAnatomicStructureModifier_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeMeaning))[SAFE_OFFSET(0)] AS primaryAnatomicStructure_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeMeaning))[SAFE_OFFSET(0)] AS primaryAnatomicStructureModifier_code_str,
 
-    ARRAY_AGG(DISTINCT(CONCAT(OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY 1)[SAFE_OFFSET(0)] AS illuminationType_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY CONCAT(OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning))[SAFE_OFFSET(0)] AS illuminationType_code_str,
 
-    ARRAY_AGG(DISTINCT(CONCAT(AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY 1)[SAFE_OFFSET(0)] AS admittingDiagnosis_code_str
+    ARRAY_AGG(DISTINCT(CONCAT(AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS ORDER BY CONCAT(AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", AdmittingDiagnosesCodeSequence[SAFE_OFFSET(0)].CodeMeaning))[SAFE_OFFSET(0)] AS admittingDiagnosis_code_str
 
 
   FROM
@@ -62,7 +62,7 @@ SpecimenPreparationSequence_unnested AS (
     slide_embedding AS (
     SELECT
       SeriesInstanceUID,
-      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) ORDER BY 1) as embeddingMedium_code_str
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) as embeddingMedium_code_str
     FROM SpecimenPreparationSequence_unnested
     WHERE (cnc_csd = 'SCT' and cnc_val = '430863003') -- CodeMeaning is 'Embedding medium'
     GROUP BY SeriesInstanceUID
@@ -71,7 +71,7 @@ SpecimenPreparationSequence_unnested AS (
     slide_fixative AS (
     SELECT
       SeriesInstanceUID,
-      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY 1) as tissueFixative_code_str
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) as tissueFixative_code_str
     FROM SpecimenPreparationSequence_unnested
     WHERE (cnc_csd = 'SCT' and cnc_val = '430864009') -- CodeMeaning is 'Tissue Fixative'
     GROUP BY SeriesInstanceUID
@@ -80,7 +80,7 @@ SpecimenPreparationSequence_unnested AS (
     slide_staining AS (
     SELECT
       SeriesInstanceUID,
-      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY 1) as staining_usingSubstance_code_str,
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) as staining_usingSubstance_code_str,
     FROM SpecimenPreparationSequence_unnested
     WHERE (cnc_csd = 'SCT' and cnc_val = '424361007') -- CodeMeaning is 'Using substance'
     GROUP BY SeriesInstanceUID
@@ -185,5 +185,5 @@ WHERE
   Modality = "SM"
 ORDER BY
   primaryAnatomicStructure_CodeMeaning,
-  staining_usingSubstance_CodeMeaning,
+  staining_usingSubstance_CodeMeaning[SAFE_OFFSET(0)],
   collection_id
diff --git a/assets/sm_instance_index.sql b/assets/sm_instance_index.sql
index f116a6a..61cbf56 100644
--- a/assets/sm_instance_index.sql
+++ b/assets/sm_instance_index.sql
@@ -24,7 +24,7 @@ WITH
   slide_embedding AS (
   SELECT
     SOPInstanceUID,
-    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) ORDER BY 1) AS embeddingMedium_code_str
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm,":",ccs_csd,":",ccs_val)) AS embeddingMedium_code_str
   FROM
     SpecimenPreparationSequence_unnested
   WHERE
@@ -35,7 +35,7 @@ WITH
   slide_fixative AS (
   SELECT
     SOPInstanceUID,
-    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY 1) AS tissueFixative_code_str
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) AS tissueFixative_code_str
   FROM
     SpecimenPreparationSequence_unnested
   WHERE
@@ -46,7 +46,7 @@ WITH
   slide_staining AS (
   SELECT
     SOPInstanceUID,
-    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY 1) AS staining_usingSubstance_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) ORDER BY CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val)) AS staining_usingSubstance_code_str,
   FROM
     SpecimenPreparationSequence_unnested
   WHERE
@@ -160,6 +160,6 @@ ON
 WHERE
   dicom_all.Modality="SM"
 ORDER BY
-  staining_usingSubstance_CodeMeaning,
+  staining_usingSubstance_CodeMeaning[SAFE_OFFSET(0)],
   TransferSyntaxUID,
   SeriesInstanceUID
diff --git a/scripts/gdc/idc_gdc_selection.sql b/scripts/gdc/idc_gdc_selection.sql
index a23c844..4a1f713 100644
--- a/scripts/gdc/idc_gdc_selection.sql
+++ b/scripts/gdc/idc_gdc_selection.sql
@@ -6,7 +6,7 @@ WITH
       StudyInstanceUID,
       ANY_VALUE(StudyDate) AS StudyDate,
       ANY_VALUE(StudyDescription) AS StudyDescription,
-      ARRAY_AGG(DISTINCT Modality ORDER BY 1) AS Modalities
+      ARRAY_AGG(DISTINCT Modality ORDER BY Modality) AS Modalities
     FROM
       `bigquery-public-data.idc_current.dicom_all`
     WHERE