Skip to content

Commit 5e818da

Browse files
fedorovfedorov
committed
eliminate redundancy, assuming there is one instance per ANN series
1 parent ad8155b commit 5e818da

2 files changed

Lines changed: 23 additions & 205 deletions

File tree

‎assets/ann_group_index.sql‎

Lines changed: 9 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,15 @@
11
# table-description:
22
# This table contains detailed metadata about individual annotation groups within
3-
# Microscopy Bulk Simple Annotations (ANN) instances in IDC. Each row corresponds to
3+
# Microscopy Bulk Simple Annotations (ANN) series in IDC. Each row corresponds to
44
# a single annotation group, providing granular information about the graphic type,
55
# number of annotations, property codes, and algorithm details. This table can be
66
# joined with ann_index using SeriesInstanceUID for series-level context.
7-
8-
WITH
9-
ann_instances AS (
10-
SELECT
11-
SOPInstanceUID,
12-
SeriesInstanceUID,
13-
AnnotationCoordinateType,
14-
AnnotationGroupSequence,
15-
ReferencedSeriesSequence,
16-
instance_size,
17-
crdc_instance_uuid
18-
FROM
19-
`bigquery-public-data.idc_v23.dicom_all`
20-
WHERE
21-
# Microscopy Bulk Simple Annotations SOP Class UID - more reliable than Modality = "ANN"
22-
SOPClassUID = "1.2.840.10008.5.1.4.1.1.91.1"
23-
),
24-
25-
-- Get referenced SOP instance for each ANN instance
26-
referenced_instances AS (
27-
SELECT
28-
ann_instances.SOPInstanceUID AS ann_SOPInstanceUID,
29-
ref_instance.ReferencedSOPInstanceUID
30-
FROM
31-
ann_instances
32-
CROSS JOIN
33-
UNNEST(ann_instances.ReferencedSeriesSequence) AS ref_series
34-
CROSS JOIN
35-
UNNEST(ref_series.ReferencedInstanceSequence) AS ref_instance
36-
)
7+
# Note: ANN series are assumed to contain a single instance.
378

389
SELECT
39-
# description:
40-
# DICOM SOPInstanceUID identifier of the annotation instance
41-
ann_instances.SOPInstanceUID,
42-
4310
# description:
4411
# DICOM SeriesInstanceUID for joining with ann_index and idc_index
45-
ann_instances.SeriesInstanceUID,
12+
ann.SeriesInstanceUID,
4613

4714
# description:
4815
# sequential number identifying this annotation group as defined in DICOM AnnotationGroupNumber attribute
@@ -68,10 +35,6 @@ SELECT
6835
# type of graphic used for annotations (POINT, POLYLINE, POLYGON, ELLIPSE, RECTANGLE) as defined in DICOM GraphicType attribute
6936
group_item.GraphicType,
7037

71-
# description:
72-
# coordinate type (2D or 3D) as defined in DICOM AnnotationCoordinateType attribute
73-
ann_instances.AnnotationCoordinateType,
74-
7538
# description:
7639
# annotation property category code tuple (CodingSchemeDesignator:CodeValue) from DICOM AnnotationPropertyCategoryCodeSequence
7740
CONCAT(
@@ -97,38 +60,12 @@ SELECT
9760
# description:
9861
# name of the algorithm from DICOM AlgorithmName attribute in AnnotationGroupAlgorithmIdentificationSequence
9962
# (when AnnotationGroupGenerationType is AUTOMATIC)
100-
group_item.AnnotationGroupAlgorithmIdentificationSequence[SAFE_OFFSET(0)].AlgorithmName AS AlgorithmName,
101-
102-
# description:
103-
# SOPInstanceUID of a referenced image instance
104-
ANY_VALUE(referenced_instances.ReferencedSOPInstanceUID) AS ReferencedSOPInstanceUID,
105-
106-
# description:
107-
# size of the annotation instance file in bytes
108-
ann_instances.instance_size,
109-
110-
# description:
111-
# CRDC UUID for downloading this annotation instance
112-
ann_instances.crdc_instance_uuid
63+
group_item.AnnotationGroupAlgorithmIdentificationSequence[SAFE_OFFSET(0)].AlgorithmName AS AlgorithmName
11364

11465
FROM
115-
ann_instances
66+
`bigquery-public-data.idc_v23.dicom_all` AS ann
11667
CROSS JOIN
117-
UNNEST(ann_instances.AnnotationGroupSequence) AS group_item
118-
LEFT JOIN
119-
referenced_instances ON ann_instances.SOPInstanceUID = referenced_instances.ann_SOPInstanceUID
120-
GROUP BY
121-
ann_instances.SOPInstanceUID,
122-
ann_instances.SeriesInstanceUID,
123-
group_item.AnnotationGroupNumber,
124-
group_item.AnnotationGroupUID,
125-
group_item.AnnotationGroupLabel,
126-
group_item.AnnotationGroupGenerationType,
127-
group_item.NumberOfAnnotations,
128-
group_item.GraphicType,
129-
ann_instances.AnnotationCoordinateType,
130-
group_item.AnnotationPropertyCategoryCodeSequence,
131-
group_item.AnnotationPropertyTypeCodeSequence,
132-
group_item.AnnotationGroupAlgorithmIdentificationSequence,
133-
ann_instances.instance_size,
134-
ann_instances.crdc_instance_uuid
68+
UNNEST(ann.AnnotationGroupSequence) AS group_item
69+
WHERE
70+
# Microscopy Bulk Simple Annotations SOP Class UID - more reliable than Modality = "ANN"
71+
SOPClassUID = "1.2.840.10008.5.1.4.1.1.91.1"

‎assets/ann_index.sql‎

Lines changed: 14 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -1,154 +1,35 @@
11
# table-description:
22
# This table contains metadata about the Microscopy Bulk Simple Annotations (ANN) series
33
# available in IDC. Each row corresponds to a DICOM series containing annotations, and
4-
# includes attributes such as the annotation coordinate type, number of annotation groups,
5-
# graphic types used, and references to the annotated image series. This table can be
6-
# joined with the main index table using the SeriesInstanceUID column.
7-
8-
WITH
9-
-- Base ANN series data
10-
ann_base AS (
11-
SELECT
12-
SeriesInstanceUID,
13-
StudyInstanceUID,
14-
SOPInstanceUID,
15-
AnnotationCoordinateType,
16-
ContentLabel,
17-
ContentDescription,
18-
AnnotationGroupSequence,
19-
ReferencedSeriesSequence
20-
FROM
21-
`bigquery-public-data.idc_v23.dicom_all`
22-
WHERE
23-
# Microscopy Bulk Simple Annotations SOP Class UID - more reliable than Modality = "ANN"
24-
SOPClassUID = "1.2.840.10008.5.1.4.1.1.91.1"
25-
),
26-
27-
-- Unnest AnnotationGroupSequence to get group-level details
28-
annotation_groups AS (
29-
SELECT
30-
ann_base.SeriesInstanceUID,
31-
group_item.AnnotationGroupNumber,
32-
group_item.AnnotationGroupLabel,
33-
group_item.AnnotationGroupGenerationType,
34-
group_item.NumberOfAnnotations,
35-
group_item.GraphicType,
36-
group_item.AnnotationGroupAlgorithmIdentificationSequence[SAFE_OFFSET(0)].AlgorithmName AS AlgorithmName,
37-
CONCAT(
38-
group_item.AnnotationPropertyCategoryCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator, ":",
39-
group_item.AnnotationPropertyCategoryCodeSequence[SAFE_OFFSET(0)].CodeValue
40-
) AS PropertyCategory_code,
41-
group_item.AnnotationPropertyCategoryCodeSequence[SAFE_OFFSET(0)].CodeMeaning AS PropertyCategory_CodeMeaning,
42-
CONCAT(
43-
group_item.AnnotationPropertyTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator, ":",
44-
group_item.AnnotationPropertyTypeCodeSequence[SAFE_OFFSET(0)].CodeValue
45-
) AS PropertyType_code,
46-
group_item.AnnotationPropertyTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning AS PropertyType_CodeMeaning
47-
FROM
48-
ann_base
49-
CROSS JOIN
50-
UNNEST(ann_base.AnnotationGroupSequence) AS group_item
51-
),
52-
53-
-- Get referenced series information
54-
referenced_series AS (
55-
SELECT
56-
ann_base.SeriesInstanceUID AS ann_SeriesInstanceUID,
57-
ref_series.SeriesInstanceUID AS referenced_SeriesInstanceUID,
58-
STRING_AGG(DISTINCT ref_instance.ReferencedSOPClassUID, ",") AS ReferencedSOPClassUIDs
59-
FROM
60-
ann_base
61-
CROSS JOIN
62-
UNNEST(ann_base.ReferencedSeriesSequence) AS ref_series
63-
CROSS JOIN
64-
UNNEST(ref_series.ReferencedInstanceSequence) AS ref_instance
65-
GROUP BY
66-
ann_base.SeriesInstanceUID, ref_series.SeriesInstanceUID
67-
),
68-
69-
-- Aggregate at series level
70-
series_aggregated AS (
71-
SELECT
72-
SeriesInstanceUID,
73-
COUNT(DISTINCT AnnotationGroupNumber) AS total_annotation_groups,
74-
SUM(NumberOfAnnotations) AS total_annotations,
75-
STRING_AGG(DISTINCT GraphicType, ",") AS GraphicTypes,
76-
STRING_AGG(DISTINCT AnnotationGroupGenerationType, ",") AS AnnotationGenerationTypes,
77-
ARRAY_AGG(DISTINCT AnnotationGroupLabel IGNORE NULLS) AS AnnotationGroupLabels,
78-
ARRAY_AGG(DISTINCT PropertyCategory_CodeMeaning IGNORE NULLS) AS AnnotationPropertyCategories,
79-
ARRAY_AGG(DISTINCT PropertyType_CodeMeaning IGNORE NULLS) AS AnnotationPropertyTypes,
80-
STRING_AGG(DISTINCT AlgorithmName, ",") AS AlgorithmNames
81-
FROM
82-
annotation_groups
83-
GROUP BY
84-
SeriesInstanceUID
85-
)
4+
# includes attributes such as the annotation coordinate type and references to the
5+
# annotated image series. For detailed group-level information (counts, graphic types,
6+
# property codes), join with ann_group_index using SeriesInstanceUID. This table can be
7+
# joined with the main idc_index table using the SeriesInstanceUID column.
8+
# Note: ANN series are assumed to contain a single instance.
869

8710
SELECT
8811
# description:
8912
# DICOM SeriesInstanceUID identifier of the annotation series
90-
ann_base.SeriesInstanceUID,
13+
ann.SeriesInstanceUID,
9114

9215
# description:
9316
# coordinate type of the annotations (2D or 3D) as defined in DICOM AnnotationCoordinateType attribute
94-
ANY_VALUE(ann_base.AnnotationCoordinateType) AS AnnotationCoordinateType,
95-
96-
# description:
97-
# total number of annotation groups in this series
98-
ANY_VALUE(series_aggregated.total_annotation_groups) AS total_annotation_groups,
99-
100-
# description:
101-
# total number of individual annotations across all groups in this series
102-
ANY_VALUE(series_aggregated.total_annotations) AS total_annotations,
103-
104-
# description:
105-
# comma-separated list of distinct graphic types used (POINT, POLYLINE, POLYGON, ELLIPSE, RECTANGLE),
106-
# aggregated from DICOM GraphicType attribute across all annotation groups
107-
ANY_VALUE(series_aggregated.GraphicTypes) AS GraphicTypes,
108-
109-
# description:
110-
# comma-separated list of annotation generation types (MANUAL or AUTOMATIC),
111-
# aggregated from DICOM AnnotationGroupGenerationType attribute
112-
ANY_VALUE(series_aggregated.AnnotationGenerationTypes) AS AnnotationGenerationTypes,
113-
114-
# description:
115-
# array of annotation group labels from DICOM AnnotationGroupLabel attribute
116-
ANY_VALUE(series_aggregated.AnnotationGroupLabels) AS AnnotationGroupLabels,
117-
118-
# description:
119-
# array of annotation property category code meanings from DICOM AnnotationPropertyCategoryCodeSequence
120-
ANY_VALUE(series_aggregated.AnnotationPropertyCategories) AS AnnotationPropertyCategories,
121-
122-
# description:
123-
# array of annotation property type code meanings from DICOM AnnotationPropertyTypeCodeSequence
124-
ANY_VALUE(series_aggregated.AnnotationPropertyTypes) AS AnnotationPropertyTypes,
125-
126-
# description:
127-
# comma-separated algorithm names from DICOM AlgorithmName attribute
128-
# in AnnotationGroupAlgorithmIdentificationSequence (when applicable)
129-
ANY_VALUE(series_aggregated.AlgorithmNames) AS AlgorithmNames,
17+
ann.AnnotationCoordinateType,
13018

13119
# description:
13220
# content label as defined in DICOM ContentLabel attribute
133-
ANY_VALUE(ann_base.ContentLabel) AS ContentLabel,
21+
ann.ContentLabel,
13422

13523
# description:
13624
# content description as defined in DICOM ContentDescription attribute
137-
ANY_VALUE(ann_base.ContentDescription) AS ContentDescription,
138-
139-
# description:
140-
# comma-separated DICOM SOP Class UIDs of the referenced images
141-
ANY_VALUE(referenced_series.ReferencedSOPClassUIDs) AS ReferencedSOPClassUIDs,
25+
ann.ContentDescription,
14226

14327
# description:
14428
# SeriesInstanceUID of the referenced image series that the annotations apply to
145-
ANY_VALUE(referenced_series.referenced_SeriesInstanceUID) AS referenced_SeriesInstanceUID
29+
ReferencedSeriesSequence[SAFE_OFFSET(0)].SeriesInstanceUID AS referenced_SeriesInstanceUID
14630

14731
FROM
148-
ann_base
149-
LEFT JOIN
150-
series_aggregated ON ann_base.SeriesInstanceUID = series_aggregated.SeriesInstanceUID
151-
LEFT JOIN
152-
referenced_series ON ann_base.SeriesInstanceUID = referenced_series.ann_SeriesInstanceUID
153-
GROUP BY
154-
ann_base.SeriesInstanceUID
32+
`bigquery-public-data.idc_v23.dicom_all` AS ann
33+
WHERE
34+
# Microscopy Bulk Simple Annotations SOP Class UID - more reliable than Modality = "ANN"
35+
SOPClassUID = "1.2.840.10008.5.1.4.1.1.91.1"

0 commit comments

Comments
 (0)