Skip to content

Commit 00358f2

Browse files
committed
perf: Fix O(n²) correlated subquery in analyse_external_idents
Replace unnecessary correlated subquery for JSONB merging with simple concatenation. The subquery was executing for every row (1M+ times), causing the import to hang for 9+ hours. The 'intelligent merging' was unnecessary because base_error_jsonb and unstable_identifier_errors_jsonb use different keys - they check mutually exclusive error conditions for the same identifier column.
1 parent b67cb0e commit 00358f2

File tree

1 file changed

+4
-21
lines changed

1 file changed

+4
-21
lines changed

migrations/20250429100000_import_job_procedures_for_external_idents.up.sql

Lines changed: 4 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -625,27 +625,10 @@ BEGIN
625625
PropagatedErrors AS (
626626
SELECT
627627
od.*,
628-
-- Merge base_error_jsonb and unstable_identifier_errors_jsonb intelligently:
629-
-- If a key exists in both, concatenate the messages with "; " separator
630-
(
631-
SELECT jsonb_object_agg(
632-
key,
633-
CASE
634-
WHEN od.base_error_jsonb ? key AND od.unstable_identifier_errors_jsonb ? key
635-
THEN (od.base_error_jsonb->>key) || '; ' || (od.unstable_identifier_errors_jsonb->>key)
636-
WHEN od.base_error_jsonb ? key
637-
THEN od.base_error_jsonb->>key
638-
ELSE od.unstable_identifier_errors_jsonb->>key
639-
END
640-
)
641-
FROM (
642-
SELECT DISTINCT key
643-
FROM jsonb_each(od.base_error_jsonb)
644-
UNION
645-
SELECT DISTINCT key
646-
FROM jsonb_each(COALESCE(od.unstable_identifier_errors_jsonb, '{}'::jsonb))
647-
) AS all_keys(key)
648-
) as final_error_jsonb,
628+
-- Simple concatenation: base_error_jsonb and unstable_identifier_errors_jsonb use different keys
629+
-- (they check different error conditions that are mutually exclusive for the same identifier column).
630+
-- The previous correlated subquery for "intelligent merging" was unnecessary and caused O(n²) performance.
631+
(COALESCE(od.base_error_jsonb, '{}'::jsonb) || COALESCE(od.unstable_identifier_errors_jsonb, '{}'::jsonb)) as final_error_jsonb,
649632
-- Check if any row within the same new entity has an error.
650633
BOOL_OR((od.base_error_jsonb || COALESCE(od.unstable_identifier_errors_jsonb, '{}'::jsonb)) != '{}'::jsonb)
651634
OVER (PARTITION BY CASE WHEN od.final_lu_id IS NULL AND od.final_est_id IS NULL THEN od.entity_signature ELSE jsonb_build_object('data_row_id', od.data_row_id) END) as entity_has_error,

0 commit comments

Comments
 (0)