Merge pull request #2162 from effigies/feat/column-unification

[ENH] Deprecate 89+ string for default age column, increase expressiveness of column definitions in sidecar files

Author: rwblair

src/common-principles.md

@@ -545,16 +545,21 @@ and a guide for using macros can be found at
             "RECOMMENDED",
             "The description of the column.",
         ),
+        "Format": "OPTIONAL",
         "Levels": "RECOMMENDED",
         "Units": "RECOMMENDED",
         "Delimiter": "OPTIONAL",
         "TermURL": "RECOMMENDED",
         "HED": "OPTIONAL",
+        "Maximum": "OPTIONAL",
+        "Minimum": "OPTIONAL",
    }
 ) }}
 
 Please note that while both `Units` and `Levels` are RECOMMENDED, typically only one
 of these two fields would be specified for describing a single TSV file column.
+In the absence of `Format`, `Units` implies the column contains numeric values,
+and `Levels` implies the column contains strings.
 
 Example:
 
@@ -563,6 +568,7 @@ Example:
   "test": {
     "LongName": "Education level",
     "Description": "Education level, self-rated by participant",
+    "Format": "integer",
     "Levels": {
       "1": "Finished primary school",
       "2": "Finished secondary school",
@@ -572,7 +578,9 @@ Example:
   },
   "bmi": {
     "LongName": "Body mass index",
+    "Format": "number",
     "Units": "kg/m^2",
+    "Minimum": 0,
     "TermURL": "https://purl.bioontology.org/ontology/SNOMEDCT/60621009"
   }
 }

src/schema/objects/columns.yaml

@@ -38,12 +38,14 @@ age:
   description: |
     Numeric value in years (float or integer value).
 
-    It is recommended to tag participant ages that are 89 or higher as 89+,
-    for privacy purposes.
+    For privacy purposes, participant ages should be capped at 89.
+    Using "89+" for ages above 88 is DEPRECATED.
   definition: {
     "LongName": "Subject age",
     "Description": "Subject age in postnatal years",
+    "Format": "number",
     "Units": "year",
+    "Maximum": 89,
   }
 cardiac:
   name: cardiac
@@ -52,6 +54,7 @@ cardiac:
     continuous pulse measurement
   definition: {
     "Description": "continuous pulse measurement",
+    "Format": "number",
     "Units": "mV"
   }
 channel:
@@ -101,8 +104,7 @@ detector_type:
   display_name: Detector Type
   description: |
     The type of detector. Only to be used if the field `DetectorType` in `*_nirs.json` is set to `mixed`.
-  anyOf:
-    - type: string
+  type: string
 derived_from:
   name: derived_from
   display_name: Derived from
@@ -176,15 +178,14 @@ group__channel:
     Which group of channels (grid/strip/seeg/depth) this channel belongs to.
     This is relevant because one group has one cable-bundle and noise can be shared.
     This can be a name or number.
-  anyOf:
-    - type: string
-    - type: number
+  type: string
 handedness:
   name: handedness
   display_name: Subject handedness
   definition: {
     "LongName": "Subject handedness",
     "Description": "String value indicating one of \"left\", \"right\", or \"ambidextrous\".",
+    "Format": "string",
     "Levels": {
       "left": "Left-handed",
       "l": "Left-handed",
@@ -362,7 +363,8 @@ pathology:
     The pathology may be specified in either `samples.tsv` or
     `sessions.tsv`, depending on whether the pathology changes over time.
   definition: {
-    "Description": "Description of the pathology of the sample or type of control."
+    "Description": "Description of the pathology of the sample or type of control.",
+    "Format": "string",
   }
 participant_id:
   name: participant_id
@@ -418,6 +420,7 @@ respiratory:
     continuous breathing measurement
   definition: {
     "Description": "continuous measurements by respiration belt",
+    "Format": "number",
     "Units": "mV"
   }
 response_time:
@@ -483,6 +486,7 @@ sex:
   definition: {
     "LongName": "sex",
     "Description": "String value indicating phenotypical sex.",
+    "Format": "string",
     "Levels": {
       "F": "Female",
       "FEMALE": "Female",
@@ -537,8 +541,7 @@ source__optodes:
   display_name: Source type
   description: |
     The type of source. Only to be used if the field `SourceType` in `*_nirs.json` is set to `mixed`.
-  anyOf:
-    - type: string
+  type: string
 species:
   name: species
   display_name: Species
@@ -550,7 +553,8 @@ species:
     `homo sapiens`.
   definition: {
     "Description":
-      "binomial species name from the NCBI Taxonomy (https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi)"
+      "Latin binomial species name from the NCBI Taxonomy (https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi).",
+    "Format": "string",
   }
 status:
   name: status
@@ -592,7 +596,8 @@ strain:
     For species different from `homo sapiens`, string value indicating
     the strain of the species, for example: `C57BL/6J`.
   definition: {
-    "Description": "name of the strain of the species"
+    "Description": "Name of the strain of the species.",
+    "Format": "string",
   }
 strain_rrid:
   name: strain_rrid
@@ -635,6 +640,7 @@ trigger:
     continuous measurement of the scanner trigger signal
   definition: {
     "Description": "continuous measurement of the scanner trigger signal",
+    "Format": "number",
     "Units": "arbitrary"
   }
 # type column in channels.tsv files

src/schema/objects/metadata.yaml

@@ -1199,6 +1199,39 @@ FlipAngle:
         unit: degree
         exclusiveMinimum: 0
         maximum: 360
+Format:
+  name: Format
+  display_name: Value format
+  description: |
+    Permitted formats for values in the described column.
+  type: string
+  # Keep synced with objects.formats
+  enum:
+    # Type formats
+    - string
+    - number
+    - integer
+    - boolean
+    # Numeric/alphanumeric strings
+    - index
+    - label
+    # Dates/times
+    - date
+    - datetime
+    - time
+    # Units
+    - unit
+    # URIs
+    - uri
+    - rrid
+    - bids_uri
+    # Paths
+    - dataset_relative
+    - file_relative
+    - participant_relative
+    - stimuli_relative
+    # Miscellaneous
+    - hed_version
 FrameAcquisitionDuration:
   name: FrameAcquisitionDuration
   display_name: Frame Acquisition Duration
@@ -2153,6 +2186,12 @@ MaxMovement:
     as measured by the head localization coils (for example, `4.8`).
   type: number
   unit: mm
+Maximum:
+  name: Maximum
+  display_name: Maximum value
+  description: |
+    Maximum value a column entry is permitted to have.
+  type: number
 MeasurementToolMetadata:
   name: MeasurementToolMetadata
   display_name: Measurement Tool Metadata
@@ -2191,6 +2230,12 @@ MetaboliteRecoveryCorrectionApplied:
     If `true`, the `hplc_recovery_fractions` column MUST be present in the
     corresponding `*_blood.tsv` file.
   type: boolean
+Minimum:
+  name: Minimum
+  display_name: Minimum value
+  description: |
+    Minimum value a column entry is permitted to have.
+  type: number
 MiscChannelCount:
   name: MiscChannelCount
   display_name: Misc Channel Count

tools/schemacode/src/bidsschematools/tests/test_schema.py

@@ -182,6 +182,11 @@ def test_formats(schema_obj):
             )
 
 
+def test_format_consistency(schema_obj):
+    """Test that the "Format" field is consistent with objects.formats."""
+    assert set(schema_obj.objects.metadata.Format.enum) == schema_obj.objects.formats.keys()
+
+
 def test_dereferencing():
     orig = {
         "ReferencedObject": {