Skip to content

Commit 0cf12eb

Browse files
mikixmikix
authored
Merge pull request #70 from smart-on-fhir/mikix/docref-quirks
Add some missing US Core checks now that we have attachment info
2 parents bdbe8ba + 37d3d3b commit 0cf12eb

16 files changed

+54
-77
lines changed

‎cumulus_library_data_metrics/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
"""Data Metrics study for Cumulus Library"""
22

3-
__version__ = "6.1.0"
3+
__version__ = "7.0.0"

‎cumulus_library_data_metrics/meta/version.jinja

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
CREATE TABLE {{ study_prefix }}__meta_version AS
2-
SELECT 1 AS data_package_version;
2+
SELECT 2 AS data_package_version;

‎cumulus_library_data_metrics/q_valid_us_core_v4/README.md

-15
Original file line numberDiff line numberDiff line change
@@ -55,21 +55,6 @@ All possible rows for the resource in question.
5555
In the case of Observation profiles,
5656
it's just the count of rows with the category in question.
5757

58-
### A note on the DiagnosticReport profile
59-
60-
The DiagnosticReport profiles mark `presentedForm` as "Must Support".
61-
62-
But since Cumulus ETL strips these fields as possible PHI,
63-
this metric does not examine them.
64-
65-
### A note on the DocumentReference profile
66-
67-
The DocumentReference profile requires one or both of `content.attachment.data`
68-
and `content.attachment.url`.
69-
70-
But since Cumulus ETL strips these fields as PHI,
71-
this metric does not require them.
72-
7358
### A note on Observation profiles
7459

7560
Unlike the other resources, which check all rows, Observations are kind of a wild

‎cumulus_library_data_metrics/us_core_v4/diagnosticreport_lab_must_support.jinja

-3
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22
-- There is another non-lab-report-specific profile,
33
-- but this is the DiagnosticReport profile that covers the LAB rows.
44

5-
-- CUMULUS-QUIRK
6-
-- Doesn't look for "presentedForm", because Cumulus strips that field.
7-
85
{% import 'us_core_v4/utils.jinja' as core_utils %}
96

107
WITH

‎cumulus_library_data_metrics/us_core_v4/diagnosticreport_note_must_support.jinja

+3-4
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22
-- There is another lab-report-specific profile,
33
-- but this is the "base DiagnosticReport" profile that covers the bare minimum.
44

5-
-- CUMULUS-QUIRK
6-
-- Doesn't look for "presentedForm", because Cumulus strips that field.
7-
85
{% import 'us_core_v4/utils.jinja' as core_utils %}
96

107
WITH
@@ -15,7 +12,8 @@ tmp_simplified AS (
1512
id,
1613
{{ utils.is_reference_of_type('encounter', 'Encounter') }} AS valid_encounter,
1714
issued IS NOT NULL AS valid_issued,
18-
performer IS NOT NULL AS valid_performer
15+
performer IS NOT NULL AS valid_performer,
16+
presentedForm IS NOT NULL AS valid_presented_form
1917
FROM tmp_slice
2018
)
2119

@@ -24,6 +22,7 @@ set ns.fields = [
2422
'valid_encounter',
2523
'valid_issued',
2624
'valid_performer',
25+
'valid_presented_form',
2726
]
2827
%}
2928

‎cumulus_library_data_metrics/us_core_v4/documentreference_mandatory.jinja

+11-17
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,11 @@
33
-- CUMULUS-QUIRK
44
-- Doesn't look at the us-core-6 attachment data/url check, because Cumulus strips those fields.
55

6+
{% import 'attachment_utils.jinja' as attachment_utils %}
67
{% import 'us_core_v4/utils.jinja' as core_utils %}
78

89
WITH
9-
tmp_attachment_flat AS (
10-
SELECT
11-
id,
12-
13-
{% if schema["content"]["attachment"] %}
14-
u.content.attachment.contenttype
15-
{% else %}
16-
CAST(NULL AS VARCHAR) AS contenttype
17-
{% endif %}
18-
19-
FROM {{ src }},
20-
UNNEST(content) AS u (content)
21-
),
22-
10+
tmp_attachment_flat AS {{ attachment_utils.extract_attachments(src, schema) }},
2311
tmp_type_codes_flat AS {{ utils.extract_codes_flat(src, 'type')}},
2412

2513
tmp_grouped AS (
@@ -44,8 +32,12 @@ tmp_grouped AS (
4432
) AS valid_type,
4533

4634
BOOL_AND(
47-
att.contenttype IS NOT NULL
48-
) AS valid_content_type
35+
att.content_type IS NOT NULL
36+
) AS valid_content_type,
37+
38+
BOOL_AND(
39+
att.has_data OR att.has_url
40+
) AS valid_us_core_6
4941

5042
FROM {{ src }} AS src
5143
LEFT JOIN tmp_attachment_flat AS att ON att.id = src.id
@@ -63,7 +55,8 @@ tmp_simplified AS (
6355
valid_type,
6456
category IS NOT NULL AS valid_category,
6557
{{ utils.is_reference_of_type('subject', 'Patient') }} AS valid_subject,
66-
valid_content_type
58+
valid_content_type,
59+
valid_us_core_6
6760
FROM tmp_grouped
6861
)
6962

@@ -74,6 +67,7 @@ set ns.fields = [
7467
'valid_category',
7568
'valid_subject',
7669
'valid_content_type',
70+
'valid_us_core_6',
7771
]
7872
%}
7973

‎cumulus_library_data_metrics/us_core_v4/documentreference_must_support.jinja

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
WITH
99

10-
{% if schema["content"]["format"] %}
10+
{% if schema["content"]["format"]["code"] or schema["content"]["format"]["system"] %}
1111
tmp_content_flat AS (
1212
SELECT
1313
id,

‎cumulus_library_data_metrics/us_core_v4/profiles.py

+8-6
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from typing import ClassVar
44

5+
from cumulus_library_data_metrics import resource_info
56
from cumulus_library_data_metrics.base import MetricMixin
67

78

@@ -17,17 +18,14 @@ class UsCoreV4Mixin(MetricMixin):
1718
],
1819
},
1920
"DocumentReference": {
20-
"content": [
21-
"attachment",
22-
"format",
23-
],
2421
"context": {
2522
"encounter": {},
2623
"period": {
2724
"start": {},
2825
"end": {},
2926
},
3027
},
28+
**resource_info.DOCREF_ATTACHMENT_SCHEMA,
3129
},
3230
"Encounter": {
3331
"hospitalization": [
@@ -89,7 +87,11 @@ def add_metric_queries(self) -> None:
8987
# - category/loinc: property to slice on for Observations
9088
# - mandatory_split: some profiles have a lot of mandatory fields, which can cause
9189
# performance issues when cubing. This is a recommended hint of how many tables to
92-
# split any mandatory cube into.
90+
# split any mandatory cube into. Note that this argument gets passed on to the metric
91+
# classes that implement this mixin, and _optionally_ used. For example, the
92+
# q_valid_us_core_v4 metric does not care about this argument, because it doesn't cube.
93+
# The c_us_core_v4_count metric *does* care about it, when cubing. It will split its
94+
# output tables into multiple tables, to keep CUBE time low.
9395

9496
# Observation is so big, that if it falls over in Athena, let's know early.
9597
# So we run these first,
@@ -109,7 +111,7 @@ def add_metric_queries(self) -> None:
109111
self.make_table(src="Condition")
110112
self.make_table(src="DiagnosticReport", name="Lab")
111113
self.make_table(src="DiagnosticReport", name="Note")
112-
self.make_table(src="DocumentReference")
114+
self.make_table(src="DocumentReference", mandatory_split=2)
113115
self.make_table(src="Encounter")
114116
self.make_table(src="Immunization")
115117
self.make_table(src="Medication")

‎tests/data/c_us_core_v4_count/general/documentreference/0.ndjson

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
{"resourceType": "DocumentReference", "id": "mandatory-plus-support", "status": "current", "category": [{"text": "X"}], "type": {"coding": [{"system": "http://loinc.org", "code": "11534-5"}]}, "subject": {"reference": "Patient/A"}, "content": [{"attachment": {"contentType": "text/plain"}, "format": {"display": "X"}}], "date": "2012-10-12", "author": [], "context": {"encounter": [{"reference": "Encounter/X"}], "period": {"end": "2015"}}}
2-
{"resourceType": "DocumentReference", "id": "mandatory-only", "status": "current", "category": [{"text": "X"}], "type": {"coding": [{"system": "http://loinc.org", "code": "11534-5"}]}, "subject": {"reference": "Patient/A"}, "content": [{"attachment": {"contentType": "text/plain"}}]}
1+
{"resourceType": "DocumentReference", "id": "mandatory-plus-support", "status": "current", "category": [{"text": "X"}], "type": {"coding": [{"system": "http://loinc.org", "code": "11534-5"}]}, "subject": {"reference": "Patient/A"}, "content": [{"attachment": {"contentType": "text/plain", "data": "xxx"}, "format": {"display": "X"}}], "date": "2012-10-12", "author": [], "context": {"encounter": [{"reference": "Encounter/X"}], "period": {"end": "2015"}}}
2+
{"resourceType": "DocumentReference", "id": "mandatory-only", "status": "current", "category": [{"text": "X"}], "type": {"coding": [{"system": "http://loinc.org", "code": "11534-5"}]}, "subject": {"reference": "Patient/A"}, "content": [{"attachment": {"contentType": "text/plain", "data": "xxx"}}]}
33
{"resourceType": "DocumentReference", "id": "support-only", "date": "2012-10-12", "author": [], "content": [{"format": {"display": "X"}}], "context": {"encounter": [{"reference": "Encounter/X"}], "period": {"start": "2014"}}}

‎tests/data/c_us_core_v4_count/general/expected_documentreference_mandatory.csv

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
cnt,year,status,valid_status,valid_type,valid_category,valid_subject,valid_content_type,valid
2-
1,cumulus__none,current,true,true,true,true,true,true
3-
1,2015,current,true,true,true,true,true,true
4-
1,2014,cumulus__none,false,false,false,false,false,false
1+
cnt,year,status,valid_status,valid_type,valid_category,valid_subject,valid_content_type,valid_us_core_6,valid
2+
1,cumulus__none,current,true,true,true,true,true,true,true
3+
1,2015,current,true,true,true,true,true,true,true
4+
1,2014,cumulus__none,false,false,false,false,false,false,false

‎tests/data/meta/general/expected_version.csv

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
data_package_version
2-
1
2+
2

‎tests/data/t_us_core_v4/docref-low-schema/expected_documentreference_mandatory.csv

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
id,valid_status,valid_type,valid_category,valid_subject,valid_content_type,valid
2-
nothing,false,false,false,false,false,false
1+
id,valid_status,valid_type,valid_category,valid_subject,valid_content_type,valid_us_core_6,valid
2+
nothing,false,false,false,false,false,false,false

‎tests/data/t_us_core_v4/mandatory/documentreference/0.ndjson

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{"resourceType": "DocumentReference", "id": "valid", "status": "current", "category": [{"text": "X"}], "type": {"coding": [{"system": "http://loinc.org", "code": "11534-5"}]}, "subject": {"reference": "Patient/A"}, "content": [{"attachment": {"contentType": "text/plain"}}]}
1+
{"resourceType": "DocumentReference", "id": "valid", "status": "current", "category": [{"text": "X"}], "type": {"coding": [{"system": "http://loinc.org", "code": "11534-5"}]}, "subject": {"reference": "Patient/A"}, "content": [{"attachment": {"contentType": "text/plain", "url": "xxx"}}]}
22
{"resourceType": "DocumentReference", "id": "type-empty", "type": {}}
33
{"resourceType": "DocumentReference", "id": "type-bad-system", "type": {"coding": [{"system": "nope", "code": "11534-5"}]}}
44
{"resourceType": "DocumentReference", "id": "type-null-flavor-good", "type": {"coding": [{"system": "http://terminology.hl7.org/CodeSystem/v3-NullFlavor", "code": "UNK"}]}}

‎tests/data/t_us_core_v4/mandatory/expected_documentreference_mandatory.csv

+14-14
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
id,valid_status,valid_type,valid_category,valid_subject,valid_content_type,valid
2-
valid,true,true,true,true,true,true
3-
type-null-flavor-good,false,true,false,false,false,false
4-
type-null-flavor-bad,false,false,false,false,false,false
5-
type-multiple-good,false,true,false,false,false,false
6-
type-multiple-bad,false,false,false,false,false,false
7-
type-empty,false,false,false,false,false,false
8-
type-bad-system,false,false,false,false,false,false
9-
subject-group,false,false,false,false,false,false
10-
nothing,false,false,false,false,false,false
11-
content-no-contenttype,false,false,false,false,false,false
12-
content-no-attachment,false,false,false,false,false,false
13-
content-multiple-good,false,false,false,false,true,false
14-
content-multiple-bad,false,false,false,false,false,false
1+
id,valid_status,valid_type,valid_category,valid_subject,valid_content_type,valid_us_core_6,valid
2+
valid,true,true,true,true,true,true,true
3+
type-null-flavor-good,false,true,false,false,false,false,false
4+
type-null-flavor-bad,false,false,false,false,false,false,false
5+
type-multiple-good,false,true,false,false,false,false,false
6+
type-multiple-bad,false,false,false,false,false,false,false
7+
type-empty,false,false,false,false,false,false,false
8+
type-bad-system,false,false,false,false,false,false,false
9+
subject-group,false,false,false,false,false,false,false
10+
nothing,false,false,false,false,false,false,false
11+
content-no-contenttype,false,false,false,false,false,true,false
12+
content-no-attachment,false,false,false,false,false,false,false
13+
content-multiple-good,false,false,false,false,true,false,false
14+
content-multiple-bad,false,false,false,false,false,false,false

‎tests/data/t_us_core_v4/must-support/diagnosticreport/notes.ndjson

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
{"resourceType": "DiagnosticReport", "id": "valid", "encounter": {"reference": "Encounter/A"}, "issued": "2022", "performer": []}
1+
{"resourceType": "DiagnosticReport", "id": "valid", "encounter": {"reference": "Encounter/A"}, "issued": "2022", "performer": [], "presentedForm": [{"contentType": "text/html"}]}
22
{"resourceType": "DiagnosticReport", "id": "wrong-encounter-type", "encounter": {"reference": "EpisodeOfCare/A"}}
33
{"resourceType": "DiagnosticReport", "id": "nothing"}

‎tests/data/t_us_core_v4/must-support/expected_diagnosticreport_note_must_support.csv

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
id,valid_encounter,valid_issued,valid_performer,valid
2-
wrong-encounter-type,false,false,false,false
3-
valid,true,true,true,true
4-
nothing,false,false,false,false
1+
id,valid_encounter,valid_issued,valid_performer,valid_presented_form,valid
2+
wrong-encounter-type,false,false,false,false,false
3+
valid,true,true,true,true,true
4+
nothing,false,false,false,false,false

0 commit comments

Comments
 (0)