From 6a7d8c74623a1858edf15bd01b69162f15c52caa Mon Sep 17 00:00:00 2001 From: David Sklar Date: Wed, 11 Mar 2026 17:49:17 +0100 Subject: [PATCH 1/5] Macrostrat all entities with mapping to all runs --- schema/development/0005-macrostrat_xdd.sql | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/schema/development/0005-macrostrat_xdd.sql b/schema/development/0005-macrostrat_xdd.sql index b616f2ea3..3b0046150 100644 --- a/schema/development/0005-macrostrat_xdd.sql +++ b/schema/development/0005-macrostrat_xdd.sql @@ -36,6 +36,20 @@ CREATE TABLE macrostrat_xdd.lookup_extraction_type ( ); ALTER TABLE macrostrat_xdd.lookup_extraction_type OWNER TO macrostrat_admin; +-- Global Macrostrat entity mapping +CREATE TABLE macrostrat_xdd.global_entity ( + global_entity_id BIGSERIAL PRIMARY KEY, + entity_table TEXT NOT NULL, + entity_id INTEGER NOT NULL, + + CONSTRAINT unique_entity UNIQUE (entity_table, entity_id) +); + +-- TODO: populate table with +-- TODO: auto sync with populated tables + +ALTER TABLE macrostrat_xdd.global_entity OWNER TO xdd_writer; + CREATE TABLE macrostrat_xdd.all_runs ( user_id uuid, "timestamp" timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, @@ -47,6 +61,14 @@ CREATE TABLE macrostrat_xdd.all_runs ( source_text_id integer NOT NULL, supersedes integer ); + +-- Add root_id +ALTER TABLE macrostrat_xdd.all_runs +ADD COLUMN root_id BIGINT +FOREIGN KEY (root_id) +REFERENCES macrostrat_xdd.global_entity(global_entity_id) +ON DELETE SET NULL; + ALTER TABLE macrostrat_xdd.all_runs OWNER TO xdd_writer; CREATE TABLE macrostrat_xdd.entity ( From 0d80c002b49252b717f52f24dd4e2ae1d6eb998f Mon Sep 17 00:00:00 2001 From: David Sklar Date: Wed, 25 Mar 2026 09:34:55 +0100 Subject: [PATCH 2/5] global entity lookup table --- schema/development/0005-macrostrat_xdd.sql | 93 +++++++++++++++++++++- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/schema/development/0005-macrostrat_xdd.sql b/schema/development/0005-macrostrat_xdd.sql index 3b0046150..146d5a463 100644 --- a/schema/development/0005-macrostrat_xdd.sql +++ b/schema/development/0005-macrostrat_xdd.sql @@ -36,20 +36,105 @@ CREATE TABLE macrostrat_xdd.lookup_extraction_type ( ); ALTER TABLE macrostrat_xdd.lookup_extraction_type OWNER TO macrostrat_admin; --- Global Macrostrat entity mapping CREATE TABLE macrostrat_xdd.global_entity ( global_entity_id BIGSERIAL PRIMARY KEY, entity_table TEXT NOT NULL, entity_id INTEGER NOT NULL, + name TEXT NOT NULL, + normalized_name TEXT NOT NULL, + CONSTRAINT unique_entity UNIQUE (entity_table, entity_id) ); --- TODO: populate table with --- TODO: auto sync with populated tables - ALTER TABLE macrostrat_xdd.global_entity OWNER TO xdd_writer; +-- INSERT SCRIPT + +INSERT INTO macrostrat_xdd.global_entity ( + entity_table, + entity_id, + name, + normalized_name +) + +SELECT + 'macrostrat.minerals', + m.id, + m.mineral, + lower(regexp_replace(m.mineral, '\s+', '', 'g')) +FROM macrostrat.minerals m + +UNION ALL + +SELECT + 'macrostrat.intervals', + i.id, + i.interval_name, + lower(regexp_replace(i.interval_name, '\s+', '', 'g')) +FROM macrostrat.intervals i + +UNION ALL + +SELECT + 'macrostrat.environs', + e.id, + e.environ, + lower(regexp_replace(e.environ, '\s+', '', 'g')) +FROM macrostrat.environs e + +UNION ALL + +SELECT + 'macrostrat.tectonics', + t.id, + t.basin_type, + lower(regexp_replace(t.basin_type, '\s+', '', 'g')) +FROM macrostrat.tectonics t + +UNION ALL + +SELECT + 'macrostrat.liths', + l.id, + l.lith, + lower(regexp_replace(l.lith, '\s+', '', 'g')) +FROM macrostrat.liths l + +UNION ALL + +SELECT + 'macrostrat.lith_atts', + la.id, + la.lith_att, + lower(regexp_replace(la.lith_att, '\s+', '', 'g')) +FROM macrostrat.lith_atts la + +UNION ALL + +SELECT + 'macrostrat.structures', + s.id, + s.structure, + lower(regexp_replace(s.structure, '\s+', '', 'g')) +FROM macrostrat.structures s + +UNION ALL + +SELECT + 'macrostrat.structure_atts', + sa.id, + sa.structure_att, + lower(regexp_replace(sa.structure_att, '\s+', '', 'g')) +FROM macrostrat.structure_atts sa + +ON CONFLICT (entity_table, entity_id) +DO UPDATE SET + name = EXCLUDED.name, + normalized_name = EXCLUDED.normalized_name; + +-- + CREATE TABLE macrostrat_xdd.all_runs ( user_id uuid, "timestamp" timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, From 7422b8eaa581b2afbe3f0a84fb2bd7284a35d335 Mon Sep 17 00:00:00 2001 From: David Sklar Date: Wed, 25 Mar 2026 18:11:28 +0100 Subject: [PATCH 3/5] feedback pg view --- schema/development/9000-macrostrat_api.sql | 71 ++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/schema/development/9000-macrostrat_api.sql b/schema/development/9000-macrostrat_api.sql index a88555591..443b3ce37 100644 --- a/schema/development/9000-macrostrat_api.sql +++ b/schema/development/9000-macrostrat_api.sql @@ -1791,6 +1791,77 @@ FROM macrostrat.strat_names where concept_id is null; ALTER TABLE macrostrat_api.autocomplete OWNER TO macrostrat_admin; +CREATE OR REPLACE VIEW macrostrat_api.feedback AS + +WITH selected_runs AS ( + SELECT * + FROM macrostrat_xdd.all_runs + WHERE user_id IS NOT NULL +), + +entities AS ( + SELECT + e.run_id, + jsonb_agg( + jsonb_build_object( + 'id', e.id, + 'text', e.name, + 'type', et.name, + 'start', e.start_index, + 'end', e.end_index + ) + ) AS entities + FROM macrostrat_xdd.entity e + LEFT JOIN macrostrat_xdd.entity_type et + ON et.id = e.entity_type_id + GROUP BY e.run_id +), + +relations AS ( + SELECT + e.run_id, + jsonb_agg( + jsonb_build_object( + 'head', r.src_entity_id, + 'tail', r.dst_entity_id + ) + ) AS relations + FROM macrostrat_xdd.relationship r + JOIN macrostrat_xdd.entity e + ON e.id = r.src_entity_id + GROUP BY e.run_id +), + +feedback_meta AS ( + SELECT + ef.feedback_id AS run_id, + ef.custom_note AS extraction_note, + eft.type AS extraction_feedback_type + FROM macrostrat_xdd.extraction_feedback ef + LEFT JOIN macrostrat_xdd.lookup_extraction_type let + ON let.note_id = ef.note_id + LEFT JOIN macrostrat_xdd.extraction_feedback_type eft + ON eft.type_id = let.type_id +) + +SELECT + sr.*, + fm.extraction_note, + fm.extraction_feedback_type, + COALESCE(ent.entities, '[]'::jsonb) AS entities, + COALESCE(rel.relations, '[]'::jsonb) AS relations + +FROM selected_runs sr +LEFT JOIN entities ent + ON ent.run_id = sr.id +LEFT JOIN relations rel + ON rel.run_id = sr.id +LEFT JOIN feedback_meta fm + ON fm.run_id = sr.id; + +ALTER TABLE macrostrat_api.feedback OWNER TO macrostrat_admin; + + GRANT USAGE ON SCHEMA macrostrat_api TO web_anon; GRANT USAGE ON SCHEMA macrostrat_api TO web_user; From 1328a9b2f34bbcd89f1f88a4d18c2a0aa8ba0eb3 Mon Sep 17 00:00:00 2001 From: David Sklar Date: Wed, 25 Mar 2026 18:28:38 +0100 Subject: [PATCH 4/5] fix view and table alters --- schema/development/0005-macrostrat_xdd.sql | 91 +--------------------- schema/development/9000-macrostrat_api.sql | 20 +++-- 2 files changed, 18 insertions(+), 93 deletions(-) diff --git a/schema/development/0005-macrostrat_xdd.sql b/schema/development/0005-macrostrat_xdd.sql index 146d5a463..2b4e8d989 100644 --- a/schema/development/0005-macrostrat_xdd.sql +++ b/schema/development/0005-macrostrat_xdd.sql @@ -49,92 +49,6 @@ CREATE TABLE macrostrat_xdd.global_entity ( ALTER TABLE macrostrat_xdd.global_entity OWNER TO xdd_writer; --- INSERT SCRIPT - -INSERT INTO macrostrat_xdd.global_entity ( - entity_table, - entity_id, - name, - normalized_name -) - -SELECT - 'macrostrat.minerals', - m.id, - m.mineral, - lower(regexp_replace(m.mineral, '\s+', '', 'g')) -FROM macrostrat.minerals m - -UNION ALL - -SELECT - 'macrostrat.intervals', - i.id, - i.interval_name, - lower(regexp_replace(i.interval_name, '\s+', '', 'g')) -FROM macrostrat.intervals i - -UNION ALL - -SELECT - 'macrostrat.environs', - e.id, - e.environ, - lower(regexp_replace(e.environ, '\s+', '', 'g')) -FROM macrostrat.environs e - -UNION ALL - -SELECT - 'macrostrat.tectonics', - t.id, - t.basin_type, - lower(regexp_replace(t.basin_type, '\s+', '', 'g')) -FROM macrostrat.tectonics t - -UNION ALL - -SELECT - 'macrostrat.liths', - l.id, - l.lith, - lower(regexp_replace(l.lith, '\s+', '', 'g')) -FROM macrostrat.liths l - -UNION ALL - -SELECT - 'macrostrat.lith_atts', - la.id, - la.lith_att, - lower(regexp_replace(la.lith_att, '\s+', '', 'g')) -FROM macrostrat.lith_atts la - -UNION ALL - -SELECT - 'macrostrat.structures', - s.id, - s.structure, - lower(regexp_replace(s.structure, '\s+', '', 'g')) -FROM macrostrat.structures s - -UNION ALL - -SELECT - 'macrostrat.structure_atts', - sa.id, - sa.structure_att, - lower(regexp_replace(sa.structure_att, '\s+', '', 'g')) -FROM macrostrat.structure_atts sa - -ON CONFLICT (entity_table, entity_id) -DO UPDATE SET - name = EXCLUDED.name, - normalized_name = EXCLUDED.normalized_name; - --- - CREATE TABLE macrostrat_xdd.all_runs ( user_id uuid, "timestamp" timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, @@ -149,7 +63,10 @@ CREATE TABLE macrostrat_xdd.all_runs ( -- Add root_id ALTER TABLE macrostrat_xdd.all_runs -ADD COLUMN root_id BIGINT +ADD COLUMN root_id BIGINT; + +ALTER TABLE macrostrat_xdd.all_runs +ADD CONSTRAINT all_runs_root_id_fkey FOREIGN KEY (root_id) REFERENCES macrostrat_xdd.global_entity(global_entity_id) ON DELETE SET NULL; diff --git a/schema/development/9000-macrostrat_api.sql b/schema/development/9000-macrostrat_api.sql index 443b3ce37..e7b714f6e 100644 --- a/schema/development/9000-macrostrat_api.sql +++ b/schema/development/9000-macrostrat_api.sql @@ -1796,7 +1796,7 @@ CREATE OR REPLACE VIEW macrostrat_api.feedback AS WITH selected_runs AS ( SELECT * FROM macrostrat_xdd.all_runs - WHERE user_id IS NOT NULL + WHERE user_id IS NOT NULL ), entities AS ( @@ -1812,6 +1812,8 @@ entities AS ( ) ) AS entities FROM macrostrat_xdd.entity e + JOIN selected_runs sr + ON sr.id = e.run_id -- 🔥 filter early LEFT JOIN macrostrat_xdd.entity_type et ON et.id = e.entity_type_id GROUP BY e.run_id @@ -1819,7 +1821,7 @@ entities AS ( relations AS ( SELECT - e.run_id, + parent.run_id, jsonb_agg( jsonb_build_object( 'head', r.src_entity_id, @@ -1827,9 +1829,11 @@ relations AS ( ) ) AS relations FROM macrostrat_xdd.relationship r - JOIN macrostrat_xdd.entity e - ON e.id = r.src_entity_id - GROUP BY e.run_id + JOIN macrostrat_xdd.entity parent + ON parent.id = r.src_entity_id + JOIN selected_runs sr + ON sr.id = parent.run_id -- 🔥 filter early + GROUP BY parent.run_id ), feedback_meta AS ( @@ -1846,6 +1850,8 @@ feedback_meta AS ( SELECT sr.*, + ge.name AS root_entity_name, + ge.entity_table AS root_entity_table, fm.extraction_note, fm.extraction_feedback_type, COALESCE(ent.entities, '[]'::jsonb) AS entities, @@ -1857,7 +1863,9 @@ LEFT JOIN entities ent LEFT JOIN relations rel ON rel.run_id = sr.id LEFT JOIN feedback_meta fm - ON fm.run_id = sr.id; + ON fm.run_id = sr.id +LEFT JOIN macrostrat_xdd.global_entity ge + ON ge.global_entity_id = sr.root_id; ALTER TABLE macrostrat_api.feedback OWNER TO macrostrat_admin; From 4c99f471d8f76c50108cc726f2a977ea22e74c34 Mon Sep 17 00:00:00 2001 From: David Sklar Date: Wed, 25 Mar 2026 18:30:00 +0100 Subject: [PATCH 5/5] format --- schema/development/0005-macrostrat_xdd.sql | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/schema/development/0005-macrostrat_xdd.sql b/schema/development/0005-macrostrat_xdd.sql index 2b4e8d989..699986195 100644 --- a/schema/development/0005-macrostrat_xdd.sql +++ b/schema/development/0005-macrostrat_xdd.sql @@ -58,19 +58,10 @@ CREATE TABLE macrostrat_xdd.all_runs ( model_job_id text, extraction_pipeline_id text, source_text_id integer NOT NULL, - supersedes integer + supersedes integer, + root_id BIGINT ); --- Add root_id -ALTER TABLE macrostrat_xdd.all_runs -ADD COLUMN root_id BIGINT; - -ALTER TABLE macrostrat_xdd.all_runs -ADD CONSTRAINT all_runs_root_id_fkey -FOREIGN KEY (root_id) -REFERENCES macrostrat_xdd.global_entity(global_entity_id) -ON DELETE SET NULL; - ALTER TABLE macrostrat_xdd.all_runs OWNER TO xdd_writer; CREATE TABLE macrostrat_xdd.entity ( @@ -422,6 +413,9 @@ ALTER TABLE ONLY macrostrat_xdd.source_text ALTER TABLE ONLY macrostrat_xdd.all_runs ADD CONSTRAINT user_id_check FOREIGN KEY (user_id) REFERENCES macrostrat_xdd.users(internal_user_id); +ALTER TABLE ONLY macrostrat_xdd.all_runs + ADD CONSTRAINT all_runs_root_id_fkey FOREIGN KEY (root_id) REFERENCES macrostrat_xdd.global_entity(global_entity_id) ON DELETE SET NULL; + GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE macrostrat_xdd.extraction_feedback TO web_anon; GRANT SELECT,INSERT,DELETE,UPDATE ON TABLE macrostrat_xdd.extraction_feedback_type TO web_anon;