feat(api,ui): Narrative ENUM migration;

rogeruiz · rogeruiz · commit c9987734560b · 2026-06-03T22:00:24.000-04:00
This patch migrates the narrative field from a boolean type to a three-value enum across the API and client. This prepares the API and the client for the work being done for #1195. This work is independent of #1317 and #1318 but closely related to them. The latter needs to be safely integrated with this patch to ensure the changes don't conflict with one another.
diff --git a/client/src/api/schemas/dbConfigurationSectionProcessing.ts b/client/src/api/schemas/dbConfigurationSectionProcessing.ts
@@ -1,4 +1,5 @@
 import type { DbSectionAction } from './dbSectionAction';
+import type { DbSectionNarrative } from './dbSectionNarrative';
 import type { DbSectionType } from './dbSectionType';
 
 /**
@@ -11,7 +12,7 @@ import type { DbSectionType } from './dbSectionType';
  */
 export interface DbConfigurationSectionProcessing {
   include: boolean;
-  narrative: boolean;
+  narrative: DbSectionNarrative;
   action: DbSectionAction;
   name: string;
   code: string;
diff --git a/client/src/api/schemas/dbSectionNarrative.ts b/client/src/api/schemas/dbSectionNarrative.ts
@@ -0,0 +1,9 @@
+
+export type DbSectionNarrative = typeof DbSectionNarrative[keyof typeof DbSectionNarrative];
+
+
+export const DbSectionNarrative = {
+  retain: 'retain',
+  remove: 'remove',
+  refine: 'refine',
+} as const;
diff --git a/client/src/api/schemas/index.ts b/client/src/api/schemas/index.ts
@@ -27,6 +27,7 @@ export * from './dbConfigurationCustomCode';
 export * from './dbConfigurationSectionProcessing';
 export * from './dbConfigurationStatus';
 export * from './dbSectionAction';
+export * from './dbSectionNarrative';
 export * from './dbSectionType';
 export * from './dbTotalConditionCodeCount';
 export * from './deleteSectionInput';
diff --git a/client/src/api/schemas/sectionUpdateInput.ts b/client/src/api/schemas/sectionUpdateInput.ts
@@ -1,11 +1,12 @@
 import type { DbSectionAction } from './dbSectionAction';
+import type { DbSectionNarrative } from './dbSectionNarrative';
 
 /**
- * Request body for modifying a section.
+ * Input model for updating a section's processing instructions.
  */
 export interface SectionUpdateInput {
   include?: boolean | null;
-  narrative?: boolean | null;
+  narrative?: DbSectionNarrative | null;
   action?: DbSectionAction | null;
   name?: string | null;
   current_code: string;
diff --git a/client/src/pages/Configurations/ConfigBuild/Sections/Sections.test.tsx b/client/src/pages/Configurations/ConfigBuild/Sections/Sections.test.tsx
@@ -12,7 +12,7 @@ const sections: DbConfigurationSectionProcessing[] = [
     action: 'refine',
     include: true,
     code: 'hist',
-    narrative: false,
+    narrative: 'remove',
     versions: ['1.1', '3.1'],
     section_type: 'standard',
   },
@@ -21,7 +21,7 @@ const sections: DbConfigurationSectionProcessing[] = [
     action: 'refine',
     include: false,
     code: 'med',
-    narrative: false,
+    narrative: 'remove',
     versions: ['1.1', '3.1', '3.1.1'],
     section_type: 'standard',
   },
@@ -30,7 +30,7 @@ const sections: DbConfigurationSectionProcessing[] = [
     action: 'retain',
     include: true,
     code: 'imm',
-    narrative: false,
+    narrative: 'remove',
     versions: ['3.1', '3.1.1'],
     section_type: 'standard',
   },
@@ -39,7 +39,7 @@ const sections: DbConfigurationSectionProcessing[] = [
     action: 'refine',
     include: true,
     code: 'mock-custom-section',
-    narrative: true,
+    narrative: 'retain',
     versions: [],
     section_type: 'custom',
   },
diff --git a/client/src/pages/Configurations/ConfigBuild/Sections/index.tsx b/client/src/pages/Configurations/ConfigBuild/Sections/index.tsx
@@ -452,10 +452,10 @@ function NarrativeSwitch({
     <Field className="flex items-center gap-3">
       <Switch
         disabled={disabled}
-        checked={currentSection.narrative}
+        checked={currentSection.narrative === 'retain'}
         onChange={(checked) => {
           updateSection(currentSection, {
-            narrative: checked,
+            narrative: checked ? 'retain' : 'remove',
           });
         }}
         aria-label={`Toggle to refine or retain the narrative block in the ${currentSection.name} section`}
diff --git a/client/src/pages/Configurations/ConfigBuild/index.test.tsx b/client/src/pages/Configurations/ConfigBuild/index.test.tsx
@@ -104,7 +104,7 @@ const baseMockConfig: GetConfigurationResponse = {
     {
       name: 'Encounters Section',
       code: 'some code',
-      narrative: false,
+      narrative: 'remove',
       include: true,
       action: 'refine',
       versions: ['1.1'],
diff --git a/refiner/app/api/v1/configurations/model.py b/refiner/app/api/v1/configurations/model.py
@@ -9,6 +9,7 @@
     DbConfigurationSectionProcessing,
     DbConfigurationStatus,
     DbSectionAction,
+    DbSectionNarrative,
     DbTotalConditionCodeCount,
     GetConfigurationResponseVersion,
 )
@@ -173,11 +174,11 @@ class DeleteSectionInput(SectionInputBase):
 
 class SectionUpdateInput(BaseModel):
     """
-    Request body for modifying a section.
+    Input model for updating a section's processing instructions.
     """
 
     include: bool | None = None
-    narrative: bool | None = None
+    narrative: DbSectionNarrative | None = None
     action: DbSectionAction | None = None
     name: str | None = None
     current_code: str
diff --git a/refiner/app/db/configurations/db.py b/refiner/app/db/configurations/db.py
@@ -89,7 +89,7 @@ async def insert_custom_section_db(
         name,
         "refine",
         True,
-        False,
+        "remove",
         versions,
         "custom",
     )
@@ -1051,9 +1051,7 @@ async def update_configuration_section_db(
         (
             prev_section.narrative,
             section_update.narrative,
-            lambda old, new: (
-                f"narrative from '{_bool_label(old)}' to '{_bool_label(new)}'"
-            ),
+            lambda old, new: (f"narrative from '{old}' to '{new}'"),
         ),
     ]
 
diff --git a/refiner/app/db/configurations/model.py b/refiner/app/db/configurations/model.py
@@ -5,6 +5,8 @@
 
 type DbSectionAction = Literal["retain", "refine"]
 
+type DbSectionNarrative = Literal["retain", "remove", "refine"]
+
 type DbConfigurationStatus = Literal["draft", "inactive", "active"]
 
 type DbSectionType = Literal["standard", "custom"]
@@ -60,7 +62,7 @@ class DbConfigurationSectionInstructions:
     """
 
     include: bool
-    narrative: bool
+    narrative: DbSectionNarrative
     action: DbSectionAction
 
 
diff --git a/refiner/app/services/configurations.py b/refiner/app/services/configurations.py
@@ -56,7 +56,7 @@ def get_default_sections() -> list[DbConfigurationSectionProcessing]:
         DbConfigurationSectionProcessing(
             name=section_spec.display_name,
             code=loinc_code,
-            narrative=True,
+            narrative="retain",
             include=True,
             action=(
                 "retain"
diff --git a/refiner/app/services/ecr/model.py b/refiner/app/services/ecr/model.py
@@ -446,7 +446,7 @@ class SectionProvenanceRecord:
     display_name: str
     include: bool
     action: str
-    narrative: bool
+    narrative: Literal["retain", "remove", "refine"]
     config_version: int | None
     source: SectionSource
     outcome: SectionOutcome = SectionOutcome.REFINED_WITH_MATCHES
diff --git a/refiner/app/services/ecr/refine.py b/refiner/app/services/ecr/refine.py
@@ -40,16 +40,16 @@
 #   - if the action is "retain" we should keep (retain) the narrative as-is
 # * DEFAULT_SECTION_INSTRUCTIONS
 #   - we should match how it shows up in the UI where narrative is set
-#     to 'false' and refine is set to 'true'
+#     to 'remove' and refine is set to 'refine'
 SKIP_SECTION_INSTRUCTIONS = DbConfigurationSectionInstructions(
     include=True,
-    narrative=True,
+    narrative="retain",
     action="retain",
 )
 
 DEFAULT_SECTION_INSTRUCTIONS = DbConfigurationSectionInstructions(
     include=True,
-    narrative=False,
+    narrative="remove",
     action="refine",
 )
 
@@ -486,7 +486,7 @@ def refine_eicr(
             # RETAINED_NARRATIVE_REMOVED below because this branch
             # reflects the spec's structural reality, not a
             # configuration choice the jurisdiction made.
-            if not section_rules.narrative:
+            if section_rules.narrative == "remove":
                 replace_narrative_with_removal_notice(
                     section=section, namespaces=HL7_NS
                 )
@@ -498,11 +498,11 @@ def refine_eicr(
             # BRANCH 2b: refinable section, jurisdiction chose to retain
             # it. honor the narrative setting.
             #
-            # the narrative=False case used to be a silent no-op (the
+            # the narrative="remove" case used to be a silent no-op (the
             # old `retain` branch was a literal `pass`); it now
             # correctly replaces the narrative with the removal notice
             # while leaving the entries untouched.
-            if not section_rules.narrative:
+            if section_rules.narrative == "remove":
                 replace_narrative_with_removal_notice(
                     section=section, namespaces=HL7_NS
                 )
@@ -515,13 +515,17 @@ def refine_eicr(
             # process_section returns a SectionRunResult describing what
             # actually happened, which _interpret_run_result maps to a
             # user-facing outcome (including the no-match policy override)
+            # TODO: When narrative="refine" is implemented, this should trigger
+            # narrative reconstruction logic. For now, treat "refine" as "remove"
+            # to avoid keeping stale narrative when entries are being filtered.
+            include_narrative = section_rules.narrative == "retain"
             run_result = process_section(
                 section=section,
                 codes_to_match=plan.codes_to_check,
                 namespaces=HL7_NS,
                 section_specification=section_specification,
                 code_system_sets=plan.code_system_sets,
-                include_narrative=section_rules.narrative,
+                include_narrative=include_narrative,
             )
             outcome = _interpret_run_result(run_result=run_result)
 
diff --git a/refiner/app/services/terminology.py b/refiner/app/services/terminology.py
@@ -1,6 +1,7 @@
 from collections import defaultdict
 from collections.abc import Iterator
 from dataclasses import dataclass, field
+from typing import Literal
 
 from pydantic import BaseModel, Field
 
@@ -242,7 +243,7 @@ class Section(BaseModel):
     code: str
     name: str
     action: str
-    narrative: bool
+    narrative: Literal["retain", "remove", "refine"]
     include: bool
 
 
diff --git a/refiner/migrations/20260603120000_narrative_bool_to_enum.sql b/refiner/migrations/20260603120000_narrative_bool_to_enum.sql
@@ -0,0 +1,59 @@
+-- migrate:up
+-- Create the new enum type for narrative
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'section_narrative') THEN
+    CREATE TYPE section_narrative AS ENUM ('retain', 'remove', 'refine');
+  END IF;
+END $$;
+
+-- Add new column with enum type
+ALTER TABLE configurations_sections
+  ADD COLUMN narrative_new section_narrative;
+
+-- Migrate data: true -> 'retain', false -> 'remove'
+UPDATE configurations_sections
+SET narrative_new = CASE
+  WHEN narrative = true THEN 'retain'::section_narrative
+  WHEN narrative = false THEN 'remove'::section_narrative
+END;
+
+-- Make the new column NOT NULL
+ALTER TABLE configurations_sections
+  ALTER COLUMN narrative_new SET NOT NULL;
+
+-- Drop the old boolean column
+ALTER TABLE configurations_sections
+  DROP COLUMN narrative;
+
+-- Rename the new column to 'narrative'
+ALTER TABLE configurations_sections
+  RENAME COLUMN narrative_new TO narrative;
+
+
+-- migrate:down
+-- Add back the boolean column
+ALTER TABLE configurations_sections
+  ADD COLUMN narrative_old boolean;
+
+-- Migrate data back: 'retain' -> true, 'remove'/'refine' -> false
+UPDATE configurations_sections
+SET narrative_old = CASE
+  WHEN narrative = 'retain' THEN true
+  ELSE false
+END;
+
+-- Make the old column NOT NULL
+ALTER TABLE configurations_sections
+  ALTER COLUMN narrative_old SET NOT NULL;
+
+-- Drop the enum column
+ALTER TABLE configurations_sections
+  DROP COLUMN narrative;
+
+-- Rename back to 'narrative'
+ALTER TABLE configurations_sections
+  RENAME COLUMN narrative_old TO narrative;
+
+-- Drop the enum type
+DROP TYPE IF EXISTS section_narrative;
diff --git a/refiner/openapi.json b/refiner/openapi.json
diff --git a/refiner/schema.sql b/refiner/schema.sql
@@ -82,6 +82,17 @@ CREATE TYPE public.section_action AS ENUM (
 );
 
 
+--
+-- Name: section_narrative; Type: TYPE; Schema: public; Owner: -
+--
+
+CREATE TYPE public.section_narrative AS ENUM (
+    'retain',
+    'remove',
+    'refine'
+);
+
+
 --
 -- Name: configurations_set_last_activated_at_on_status_change(); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -211,11 +222,11 @@ CREATE TABLE public.configurations_sections (
     name text NOT NULL,
     action public.section_action NOT NULL,
     include boolean NOT NULL,
-    narrative boolean NOT NULL,
     versions text[] DEFAULT '{}'::text[] NOT NULL,
     created_at timestamp with time zone DEFAULT now() NOT NULL,
     updated_at timestamp with time zone DEFAULT now() NOT NULL,
-    section_type public.configurations_sections_type NOT NULL
+    section_type public.configurations_sections_type NOT NULL,
+    narrative public.section_narrative CONSTRAINT configurations_sections_narrative_new_not_null NOT NULL
 );
 
 
@@ -241,7 +252,7 @@ CREATE TABLE public.events (
 CREATE TABLE public.events_custom_code_uploads (
     id uuid DEFAULT gen_random_uuid() NOT NULL,
     event_id uuid NOT NULL,
-    system_key text CONSTRAINT events_custom_code_uploads_system_not_null NOT NULL,
+    system text NOT NULL,
     code text NOT NULL,
     name text NOT NULL
 );
@@ -712,4 +723,5 @@ INSERT INTO public.schema_migrations (version) VALUES
     ('20260511160133'),
     ('20260520185510'),
     ('20260526153052'),
-    ('20260602161536');
+    ('20260602161536'),
+    ('20260603120000');

Original file line number	Diff line number	Diff line change
`@@ -104,7 +104,7 @@ const baseMockConfig: GetConfigurationResponse = {`
`104`	`104`	`{`
`105`	`105`	`name: 'Encounters Section',`
`106`	`106`	`code: 'some code',`
`107`		`- narrative: false,`
	`107`	`+ narrative: 'remove',`
`108`	`108`	`include: true,`
`109`	`109`	`action: 'refine',`
`110`	`110`	`versions: ['1.1'],`
Original file line number	Diff line number	Diff line change
`@@ -89,7 +89,7 @@ async def insert_custom_section_db(`
`89`	`89`	`name,`
`90`	`90`	`"refine",`
`91`	`91`	`True,`
`92`		`- False,`
	`92`	`+ "remove",`
`93`	`93`	`versions,`
`94`	`94`	`"custom",`
`95`	`95`	`)`
`@@ -1051,9 +1051,7 @@ async def update_configuration_section_db(`
`1051`	`1051`	`(`
`1052`	`1052`	`prev_section.narrative,`
`1053`	`1053`	`section_update.narrative,`
`1054`		`- lambda old, new: (`
`1055`		`- f"narrative from '{_bool_label(old)}' to '{_bool_label(new)}'"`
`1056`		`- ),`
	`1054`	`+ lambda old, new: (f"narrative from '{old}' to '{new}'"),`
`1057`	`1055`	`),`
`1058`	`1056`	`]`
`1059`	`1057`