Skip to content

Commit e85e34c

Browse files
jhfclaude
jhf
and
claude
committed
db: Restore original functions in legal_relationship down migration
The up migration modified two shared functions but the down migration didn't restore them. Now restores the original bodies of import.create_source_and_mappings_for_definition() and admin.validate_import_definition(). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 360029f commit e85e34c

4 files changed

+522
-175
lines changed

‎migrations/20260218215337_add_legal_relationship_import.down.sql‎

Lines changed: 349 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,353 @@ DROP PROCEDURE IF EXISTS import.analyse_legal_relationship(INT, INTEGER, TEXT);
1818
-- Note: Cannot remove enum value 'legal_relationship' from import_mode in PostgreSQL.
1919
-- The enum value will remain but be unused.
2020

21+
-- Restore import.create_source_and_mappings_for_definition() to original version
22+
-- (removes the v_has_stat_step guard — original always maps statistical variables)
23+
CREATE OR REPLACE FUNCTION import.create_source_and_mappings_for_definition(
24+
p_definition_id INT,
25+
p_source_columns TEXT[] -- Array of source column names expected in the file
26+
) RETURNS void LANGUAGE plpgsql AS $$
27+
DECLARE
28+
v_def public.import_definition;
29+
v_col_name TEXT;
30+
v_priority INT := 0;
31+
v_source_col_id INT;
32+
v_data_col_id INT;
33+
v_max_priority INT;
34+
v_col_rec RECORD;
35+
BEGIN
36+
SELECT * INTO v_def FROM public.import_definition WHERE id = p_definition_id;
37+
38+
-- Handle validity date mappings based on definition mode
39+
IF v_def.valid_time_from = 'job_provided' THEN
40+
FOR v_col_name IN VALUES ('valid_from'), ('valid_to') LOOP
41+
SELECT dc.id INTO v_data_col_id FROM public.import_data_column dc JOIN public.import_step s ON dc.step_id = s.id WHERE s.code = 'valid_time' AND dc.column_name = v_col_name || '_raw';
42+
IF v_data_col_id IS NOT NULL THEN
43+
INSERT INTO public.import_mapping (definition_id, source_expression, target_data_column_id, target_data_column_purpose)
44+
VALUES (p_definition_id, 'default', v_data_col_id, 'source_input'::public.import_data_column_purpose)
45+
ON CONFLICT (definition_id, target_data_column_id) WHERE is_ignored = false DO NOTHING;
46+
END IF;
47+
END LOOP;
48+
END IF;
49+
50+
-- Create source columns and map them
51+
FOREACH v_col_name IN ARRAY p_source_columns LOOP
52+
v_priority := v_priority + 1;
53+
INSERT INTO public.import_source_column (definition_id, column_name, priority)
54+
VALUES (p_definition_id, v_col_name, v_priority)
55+
ON CONFLICT DO NOTHING RETURNING id INTO v_source_col_id;
56+
57+
IF v_source_col_id IS NOT NULL THEN
58+
SELECT dc.id INTO v_data_col_id
59+
FROM public.import_definition_step ds
60+
JOIN public.import_data_column dc ON ds.step_id = dc.step_id
61+
WHERE ds.definition_id = p_definition_id AND dc.column_name = v_col_name || '_raw' AND dc.purpose = 'source_input';
62+
63+
IF v_data_col_id IS NOT NULL THEN
64+
INSERT INTO public.import_mapping (definition_id, source_column_id, target_data_column_id, target_data_column_purpose)
65+
VALUES (p_definition_id, v_source_col_id, v_data_col_id, 'source_input'::public.import_data_column_purpose)
66+
ON CONFLICT (definition_id, source_column_id, target_data_column_id) DO NOTHING;
67+
ELSE
68+
INSERT INTO public.import_mapping (definition_id, source_column_id, is_ignored)
69+
VALUES (p_definition_id, v_source_col_id, TRUE)
70+
ON CONFLICT (definition_id, source_column_id, target_data_column_id) WHERE target_data_column_id IS NULL DO NOTHING;
71+
END IF;
72+
END IF;
73+
END LOOP;
74+
75+
-- Dynamically add and map source columns for Statistical Variables
76+
SELECT COALESCE(MAX(priority), v_priority) INTO v_max_priority FROM public.import_source_column WHERE definition_id = p_definition_id;
77+
INSERT INTO public.import_source_column (definition_id, column_name, priority)
78+
SELECT p_definition_id, stat.code, v_max_priority + ROW_NUMBER() OVER (ORDER BY stat.priority)
79+
FROM public.stat_definition_active stat ON CONFLICT (definition_id, column_name) DO NOTHING;
80+
81+
FOR v_col_rec IN
82+
SELECT isc.id as source_col_id, isc.column_name as stat_code FROM public.import_source_column isc
83+
JOIN public.stat_definition_active sda ON isc.column_name = sda.code
84+
WHERE isc.definition_id = p_definition_id AND NOT EXISTS (
85+
SELECT 1 FROM public.import_mapping im WHERE im.definition_id = p_definition_id AND im.source_column_id = isc.id
86+
)
87+
LOOP
88+
SELECT dc.id INTO v_data_col_id FROM public.import_definition_step ds
89+
JOIN public.import_step s ON ds.step_id = s.id
90+
JOIN public.import_data_column dc ON ds.step_id = dc.step_id
91+
WHERE ds.definition_id = p_definition_id AND s.code = 'statistical_variables' AND dc.column_name = v_col_rec.stat_code || '_raw' AND dc.purpose = 'source_input';
92+
93+
IF v_data_col_id IS NOT NULL THEN
94+
INSERT INTO public.import_mapping (definition_id, source_column_id, target_data_column_id, target_data_column_purpose)
95+
VALUES (p_definition_id, v_col_rec.source_col_id, v_data_col_id, 'source_input')
96+
ON CONFLICT (definition_id, source_column_id, target_data_column_id) DO NOTHING;
97+
ELSE
98+
RAISE EXCEPTION '[Definition %] No matching source_input data column found in "statistical_variables" step for dynamically added stat source column "%".', p_definition_id, v_col_rec.stat_code;
99+
END IF;
100+
END LOOP;
101+
END;
102+
$$;
103+
104+
-- Restore admin.validate_import_definition() to original version
105+
-- (removes legal_relationship mode branch and conditional external_idents check)
106+
CREATE OR REPLACE FUNCTION admin.validate_import_definition(p_definition_id INT)
107+
RETURNS void LANGUAGE plpgsql AS $validate_import_definition$
108+
DECLARE
109+
v_definition public.import_definition;
110+
v_error_messages TEXT[] := ARRAY[]::TEXT[];
111+
v_is_valid BOOLEAN := true;
112+
v_step_codes TEXT[];
113+
v_has_time_from_context_step BOOLEAN;
114+
v_has_time_from_source_step BOOLEAN;
115+
v_has_valid_from_mapping BOOLEAN := false;
116+
v_has_valid_to_mapping BOOLEAN := false;
117+
v_source_col_rec RECORD;
118+
v_mapping_rec RECORD;
119+
v_temp_text TEXT;
120+
BEGIN
121+
SELECT * INTO v_definition FROM public.import_definition WHERE id = p_definition_id;
122+
IF NOT FOUND THEN
123+
RAISE DEBUG 'validate_import_definition: Definition ID % not found. Skipping validation.', p_definition_id;
124+
RETURN;
125+
END IF;
126+
127+
-- 1. Time Validity Method Check
128+
-- All definitions must include the 'valid_time' step to ensure uniform processing.
129+
IF NOT EXISTS (SELECT 1 FROM public.import_definition_step ids JOIN public.import_step s ON s.id = ids.step_id WHERE ids.definition_id = p_definition_id AND s.code = 'valid_time') THEN
130+
v_is_valid := false;
131+
v_error_messages := array_append(v_error_messages, 'All import definitions must include the "valid_time" step.');
132+
END IF;
133+
134+
-- The following checks ensure the mappings for 'valid_from' and 'valid_to' are consistent with the chosen time validity mode.
135+
IF v_definition.valid_time_from = 'source_columns' THEN
136+
-- Check that 'valid_from_raw' and 'valid_to_raw' are mapped from source columns.
137+
SELECT EXISTS (
138+
SELECT 1 FROM public.import_mapping im
139+
JOIN public.import_data_column idc ON im.target_data_column_id = idc.id JOIN public.import_step s ON idc.step_id = s.id
140+
WHERE im.definition_id = p_definition_id AND s.code = 'valid_time' AND idc.column_name = 'valid_from_raw' AND im.source_column_id IS NOT NULL AND im.is_ignored = FALSE
141+
) INTO v_has_valid_from_mapping;
142+
SELECT EXISTS (
143+
SELECT 1 FROM public.import_mapping im
144+
JOIN public.import_data_column idc ON im.target_data_column_id = idc.id JOIN public.import_step s ON idc.step_id = s.id
145+
WHERE im.definition_id = p_definition_id AND s.code = 'valid_time' AND idc.column_name = 'valid_to_raw' AND im.source_column_id IS NOT NULL AND im.is_ignored = FALSE
146+
) INTO v_has_valid_to_mapping;
147+
148+
IF NOT (v_has_valid_from_mapping AND v_has_valid_to_mapping) THEN
149+
v_is_valid := false;
150+
v_error_messages := array_append(v_error_messages, 'When valid_time_from="source_columns", mappings for both "valid_from_raw" and "valid_to_raw" from source columns are required.');
151+
END IF;
152+
153+
ELSIF v_definition.valid_time_from = 'job_provided' THEN
154+
-- If validity is derived from job-level parameters, the definition must map 'valid_from_raw'
155+
-- and 'valid_to_raw' to the 'default' source expression. This allows the `import_job_prepare`
156+
-- function to populate these columns from the job's `default_valid_from`/`to` fields.
157+
SELECT EXISTS (
158+
SELECT 1 FROM public.import_mapping im
159+
JOIN public.import_data_column idc ON im.target_data_column_id = idc.id JOIN public.import_step s ON idc.step_id = s.id
160+
WHERE im.definition_id = p_definition_id AND s.code = 'valid_time' AND idc.column_name = 'valid_from_raw' AND im.source_expression = 'default' AND im.is_ignored = FALSE
161+
) INTO v_has_valid_from_mapping;
162+
SELECT EXISTS (
163+
SELECT 1 FROM public.import_mapping im
164+
JOIN public.import_data_column idc ON im.target_data_column_id = idc.id JOIN public.import_step s ON idc.step_id = s.id
165+
WHERE im.definition_id = p_definition_id AND s.code = 'valid_time' AND idc.column_name = 'valid_to_raw' AND im.source_expression = 'default' AND im.is_ignored = FALSE
166+
) INTO v_has_valid_to_mapping;
167+
168+
IF NOT (v_has_valid_from_mapping AND v_has_valid_to_mapping) THEN
169+
v_is_valid := false;
170+
v_error_messages := array_append(v_error_messages, 'When valid_time_from="job_provided", mappings for both "valid_from_raw" and "valid_to_raw" using source_expression="default" are required.');
171+
END IF;
172+
173+
ELSE
174+
v_is_valid := false;
175+
v_error_messages := array_append(v_error_messages, 'valid_time_from is NULL or has an unhandled value.');
176+
END IF;
177+
178+
-- 2. Mode-specific step checks
179+
SELECT array_agg(s.code) INTO v_step_codes
180+
FROM public.import_definition_step ids
181+
JOIN public.import_step s ON ids.step_id = s.id
182+
WHERE ids.definition_id = p_definition_id;
183+
v_step_codes := COALESCE(v_step_codes, ARRAY[]::TEXT[]);
184+
185+
IF v_definition.mode = 'legal_unit' THEN
186+
IF NOT ('legal_unit' = ANY(v_step_codes)) THEN
187+
v_is_valid := false;
188+
v_error_messages := array_append(v_error_messages, 'Mode "legal_unit" requires the "legal_unit" step.');
189+
END IF;
190+
IF NOT ('enterprise_link_for_legal_unit' = ANY(v_step_codes)) THEN
191+
v_is_valid := false;
192+
v_error_messages := array_append(v_error_messages, 'Mode "legal_unit" requires the "enterprise_link_for_legal_unit" step.');
193+
END IF;
194+
ELSIF v_definition.mode = 'establishment_formal' THEN
195+
IF NOT ('establishment' = ANY(v_step_codes)) THEN
196+
v_is_valid := false;
197+
v_error_messages := array_append(v_error_messages, 'Mode "establishment_formal" requires the "establishment" step.');
198+
END IF;
199+
IF NOT ('link_establishment_to_legal_unit' = ANY(v_step_codes)) THEN
200+
v_is_valid := false;
201+
v_error_messages := array_append(v_error_messages, 'Mode "establishment_formal" requires the "link_establishment_to_legal_unit" step.');
202+
END IF;
203+
ELSIF v_definition.mode = 'establishment_informal' THEN
204+
IF NOT ('establishment' = ANY(v_step_codes)) THEN
205+
v_is_valid := false;
206+
v_error_messages := array_append(v_error_messages, 'Mode "establishment_informal" requires the "establishment" step.');
207+
END IF;
208+
IF NOT ('enterprise_link_for_establishment' = ANY(v_step_codes)) THEN
209+
v_is_valid := false;
210+
v_error_messages := array_append(v_error_messages, 'Mode "establishment_informal" requires the "enterprise_link_for_establishment" step.');
211+
END IF;
212+
ELSIF v_definition.mode = 'generic_unit' THEN
213+
-- Generic unit mode might have fewer structural step requirements.
214+
-- It still needs external_idents to find the unit, and likely statistical_variables if that's its purpose.
215+
-- For now, no specific structural checks beyond the global mandatory ones.
216+
RAISE DEBUG '[Validate Def ID %] Mode is generic_unit, skipping LU/ES specific step checks.', p_definition_id;
217+
ELSE
218+
-- This case should ideally not be reached if the mode enum is exhaustive and NOT NULL
219+
v_is_valid := false;
220+
v_error_messages := array_append(v_error_messages, format('Unknown or unhandled import mode: %L.', v_definition.mode));
221+
END IF;
222+
223+
-- Enforce unique step priorities within a definition (prevents equal-priority deadlocks in analysis scheduling)
224+
IF EXISTS (
225+
SELECT 1
226+
FROM (
227+
SELECT s.priority
228+
FROM public.import_definition_step ids
229+
JOIN public.import_step s ON s.id = ids.step_id
230+
WHERE ids.definition_id = p_definition_id
231+
GROUP BY s.priority
232+
HAVING COUNT(*) > 1
233+
) dup
234+
) THEN
235+
v_is_valid := false;
236+
v_error_messages := array_append(v_error_messages, 'import_step priorities must be unique per definition (duplicates found).');
237+
END IF;
238+
239+
-- 3. Check for mandatory steps
240+
IF NOT ('external_idents' = ANY(v_step_codes)) THEN
241+
v_is_valid := false;
242+
v_error_messages := array_append(v_error_messages, 'The "external_idents" step is mandatory.');
243+
END IF;
244+
IF NOT ('edit_info' = ANY(v_step_codes)) THEN
245+
v_is_valid := false;
246+
v_error_messages := array_append(v_error_messages, 'The "edit_info" step is mandatory.');
247+
END IF;
248+
IF NOT ('metadata' = ANY(v_step_codes)) THEN
249+
v_is_valid := false;
250+
v_error_messages := array_append(v_error_messages, 'The "metadata" step is mandatory.');
251+
END IF;
252+
253+
-- 4. Source Column and Mapping Consistency
254+
255+
-- Specific check for 'external_idents' step:
256+
-- If 'external_idents' step is included, at least one of its 'source_input' data columns must be mapped.
257+
IF 'external_idents' = ANY(v_step_codes) THEN
258+
DECLARE
259+
v_has_mapped_external_ident BOOLEAN;
260+
BEGIN
261+
SELECT EXISTS (
262+
SELECT 1 FROM public.import_mapping im
263+
JOIN public.import_data_column idc ON im.target_data_column_id = idc.id
264+
JOIN public.import_step s ON idc.step_id = s.id
265+
WHERE im.definition_id = p_definition_id
266+
AND s.code = 'external_idents'
267+
AND idc.purpose = 'source_input'
268+
AND im.is_ignored = FALSE
269+
) INTO v_has_mapped_external_ident;
270+
271+
IF NOT v_has_mapped_external_ident THEN
272+
v_is_valid := false;
273+
v_error_messages := array_append(v_error_messages, 'At least one external identifier column (e.g., tax_ident, stat_ident) must be mapped for the "external_idents" step.');
274+
END IF;
275+
END;
276+
END IF;
277+
278+
-- Specific check for 'status' step removed, as status_code mapping is now optional.
279+
-- The analyse_status procedure will handle defaults, and analyse_legal_unit/_establishment
280+
-- will error if status_id is ultimately not resolved.
281+
282+
-- Conditional check for 'data_source_code_raw' mapping:
283+
-- If import_definition.data_source_id is NULL, a mapping for 'data_source_code_raw' is required.
284+
IF v_definition.data_source_id IS NULL THEN
285+
DECLARE
286+
v_data_source_code_mapped BOOLEAN;
287+
v_data_source_code_data_column_exists BOOLEAN;
288+
BEGIN
289+
-- Check if a data_source_code_raw data column even exists for any of the definition's steps
290+
SELECT EXISTS (
291+
SELECT 1
292+
FROM public.import_definition_step ids
293+
JOIN public.import_data_column idc ON ids.step_id = idc.step_id
294+
WHERE ids.definition_id = p_definition_id
295+
AND idc.column_name = 'data_source_code_raw'
296+
AND idc.purpose = 'source_input'
297+
) INTO v_data_source_code_data_column_exists;
298+
299+
IF v_data_source_code_data_column_exists THEN
300+
-- If the data column exists, check if it's mapped
301+
SELECT EXISTS (
302+
SELECT 1 FROM public.import_mapping im
303+
JOIN public.import_data_column idc ON im.target_data_column_id = idc.id
304+
WHERE im.definition_id = p_definition_id
305+
AND idc.column_name = 'data_source_code_raw'
306+
AND idc.purpose = 'source_input'
307+
AND im.is_ignored = FALSE
308+
) INTO v_data_source_code_mapped;
309+
310+
IF NOT v_data_source_code_mapped THEN
311+
v_is_valid := false;
312+
v_error_messages := array_append(v_error_messages, 'If import_definition.data_source_id is NULL and a "data_source_code_raw" source_input data column is available for the definition''s steps, it must be mapped.');
313+
END IF;
314+
ELSE
315+
-- If data_source_id is NULL and no data_source_code_raw data column is available from steps, it's an error.
316+
v_is_valid := false;
317+
v_error_messages := array_append(v_error_messages, 'If import_definition.data_source_id is NULL, a "data_source_code_raw" source_input data column must be available via one of the definition''s steps and mapped. None found.');
318+
END IF;
319+
END;
320+
END IF;
321+
322+
-- The old generic loop checking all source_input columns for mapping is removed.
323+
-- Only specific, critical mappings are checked above.
324+
325+
FOR v_source_col_rec IN
326+
SELECT isc.column_name
327+
FROM public.import_source_column isc
328+
WHERE isc.definition_id = p_definition_id
329+
AND NOT EXISTS (
330+
SELECT 1 FROM public.import_mapping im
331+
WHERE im.definition_id = p_definition_id AND im.source_column_id = isc.id
332+
)
333+
LOOP
334+
v_is_valid := false;
335+
v_error_messages := array_append(v_error_messages, format('Unused import_source_column: "%s". It is defined but not used in any mapping.', v_source_col_rec.column_name));
336+
END LOOP;
337+
338+
FOR v_mapping_rec IN
339+
SELECT im.id as mapping_id, idc.column_name as target_col_name, s.code as target_step_code
340+
FROM public.import_mapping im
341+
JOIN public.import_data_column idc ON im.target_data_column_id = idc.id -- This JOIN implies target_data_column_id IS NOT NULL
342+
JOIN public.import_step s ON idc.step_id = s.id
343+
WHERE im.definition_id = p_definition_id
344+
AND im.is_ignored = FALSE -- Only validate non-ignored mappings for this check
345+
AND NOT EXISTS (
346+
SELECT 1 FROM public.import_definition_step ids
347+
WHERE ids.definition_id = p_definition_id AND ids.step_id = s.id
348+
)
349+
LOOP
350+
v_is_valid := false;
351+
v_error_messages := array_append(v_error_messages, format('Mapping ID %s targets data column "%s" in step "%s", but this step is not part of the definition.', v_mapping_rec.mapping_id, v_mapping_rec.target_col_name, v_mapping_rec.target_step_code));
352+
END LOOP;
353+
354+
-- Final Update
355+
IF v_is_valid THEN
356+
UPDATE public.import_definition
357+
SET valid = true, validation_error = NULL
358+
WHERE id = p_definition_id;
359+
ELSE
360+
-- Concatenate unique error messages
361+
SELECT string_agg(DISTINCT error_msg, '; ') INTO v_temp_text FROM unnest(v_error_messages) AS error_msg;
362+
UPDATE public.import_definition
363+
SET valid = false, validation_error = v_temp_text
364+
WHERE id = p_definition_id;
365+
END IF;
366+
367+
END;
368+
$validate_import_definition$;
369+
21370
END;

0 commit comments

Comments
 (0)