Merge pull request #240 from UW-Macrostrat/schema-updates

davenquinn · web-flow · commit 0e442d880fdc · 2025-12-27T02:16:54.000-05:00
Schema updates
diff --git a/cli/macrostrat/cli/data-scripts/composite-projects.sql b/cli/macrostrat/cli/data-scripts/composite-projects.sql
@@ -0,0 +1,29 @@
+DO $$
+DECLARE
+  _core_project_id integer;
+BEGIN
+
+INSERT INTO macrostrat.projects (id, project, descrip, timescale_id, is_composite, slug)
+VALUES
+  (
+   14,
+   'Core columns',
+   'The "best available" default set of regional columns, composited from other datasets.',
+   1,
+   TRUE,
+   'core'
+  )
+ON CONFLICT (id) DO NOTHING;
+
+SELECT id INTO _core_project_id
+FROM macrostrat.projects
+WHERE slug = 'core';
+
+INSERT INTO macrostrat.projects_tree (parent_id, child_id)
+SELECT _core_project_id, id
+FROM macrostrat.projects p
+WHERE p.slug IN ('north-america', 'caribbean', 'south-america', 'africa', 'eodp')
+ON CONFLICT DO NOTHING;
+
+END;
+$$ LANGUAGE plpgsql;
diff --git a/cli/macrostrat/cli/database/migrations/api_v3/__init__.py b/cli/macrostrat/cli/database/migrations/api_v3/__init__.py
@@ -12,7 +12,7 @@ class BaselineMigration(Migration):
     readiness_state = "ga"
     # Confirm that the tables created by the API v3 migrations are present
     postconditions = [
-        exists("storage", "object_group", "object"),
+        exists("storage", "object"),
         exists("maps_metadata", "ingest_process", "ingest_process_tag"),
         exists("macrostrat_auth", "user", "group"),
     ]
diff --git a/cli/macrostrat/cli/database/migrations/column_builder/00-view-drops.sql b/cli/macrostrat/cli/database/migrations/column_builder/00-view-drops.sql
@@ -2,7 +2,10 @@
 Drop all views so they can be recreated.
 TODO: automate this process.
 */
-DROP VIEW IF EXISTS macrostrat_api.projects;
+DROP VIEW IF EXISTS macrostrat_api.units CASCADE;
+DROP VIEW IF EXISTS macrostrat_api.unit_liths CASCADE;
+DROP VIEW IF EXISTS macrostrat_api.projects CASCADE;
+DROP VIEW IF EXISTS macrostrat_api.autocomplete CASCADE;
 DROP VIEW IF EXISTS macrostrat_api.cols;
 DROP VIEW IF EXISTS macrostrat_api.col_groups;
 DROP VIEW IF EXISTS macrostrat_api.environs;
@@ -11,10 +14,8 @@ DROP VIEW IF EXISTS macrostrat_api.intervals;
 DROP VIEW IF EXISTS macrostrat_api.timescales;
 DROP VIEW IF EXISTS macrostrat_api.strat_tree;
 DROP VIEW IF EXISTS macrostrat_api.refs;
-DROP VIEW IF EXISTS macrostrat_api.units;
 DROP VIEW IF EXISTS macrostrat_api.col_refs;
 DROP VIEW IF EXISTS macrostrat_api.unit_environs;
-DROP VIEW IF EXISTS macrostrat_api.unit_liths;
 DROP VIEW IF EXISTS macrostrat_api.sections;
 DROP VIEW IF EXISTS macrostrat_api.strat_names;
 DROP VIEW IF EXISTS macrostrat_api.unit_strat_names;
diff --git a/cli/macrostrat/cli/database/migrations/composite_projects/__init__.py b/cli/macrostrat/cli/database/migrations/composite_projects/__init__.py
@@ -0,0 +1,50 @@
+from pathlib import Path
+
+from macrostrat.core.migrations import Migration, _not, exists, has_columns
+
+
+def check_slug_not_nullable(db):
+    result = db.run_query(
+        """
+        SELECT is_nullable::boolean
+        FROM information_schema.columns
+        WHERE table_schema = 'macrostrat'
+          AND table_name = 'projects'
+          AND column_name = 'slug';
+        """
+    ).one_or_none()
+    if result is None:
+        return False
+    return not result.is_nullable
+
+
+success = [
+    has_columns("macrostrat", "projects", "is_composite", "slug"),
+    exists("macrostrat", "projects_tree"),
+    # Slug is not nullable
+    check_slug_not_nullable,
+]
+
+here = Path(__file__).parent
+
+
+class CompositeProjects(Migration):
+    name = "composite-projects"
+    subsystem = "columns"
+    description = "Composite projects support"
+    preconditions = [_not(a) for a in success]
+    postconditions = success
+    fixtures = [
+        here / "projects-evolution.sql",
+    ]
+
+
+class CompositeProjectFunctions(Migration):
+    name = "composite-projects-functions"
+    subsystem = "columns"
+    description = "Composite projects functions"
+    depends_on = ["composite-projects"]
+    always_apply = True
+    fixtures = [
+        here / "project-functions.sql",
+    ]
diff --git a/cli/macrostrat/cli/database/migrations/composite_projects/project-functions.sql b/cli/macrostrat/cli/database/migrations/composite_projects/project-functions.sql
@@ -0,0 +1,23 @@
+/** Recursive function to get all descendant project IDs **/
+CREATE OR REPLACE FUNCTION macrostrat.flattened_project_ids(project_ids integer[]) RETURNS integer[] AS $$
+DECLARE
+  result_ids integer[] := ARRAY[]::integer[];
+  current_ids integer[] := project_ids;
+  child_ids integer[];
+BEGIN
+  LOOP
+    EXIT WHEN array_length(current_ids, 1) IS NULL;
+    result_ids := result_ids || current_ids;
+    SELECT array_agg(pt.child_id)
+    INTO child_ids
+    FROM macrostrat.projects_tree pt
+    WHERE pt.parent_id = ANY(current_ids);
+    current_ids := child_ids;
+  END LOOP;
+  RETURN ARRAY(SELECT DISTINCT unnest(result_ids));
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+CREATE OR REPLACE FUNCTION macrostrat.core_project_ids() RETURNS integer[] AS $$
+SELECT macrostrat.flattened_project_ids(ARRAY[id]) FROM macrostrat.projects WHERE slug = 'core';
+$$ LANGUAGE sql STABLE;
diff --git a/cli/macrostrat/cli/database/migrations/composite_projects/projects-evolution.sql b/cli/macrostrat/cli/database/migrations/composite_projects/projects-evolution.sql
@@ -1,19 +1,33 @@
+ALTER SCHEMA macrostrat OWNER TO macrostrat;
+
+DROP VIEW IF EXISTS macrostrat_api.projects;
+DROP VIEW IF EXISTS macrostrat_api.col_group_with_cols;
+DROP VIEW IF EXISTS macrostrat_api.autocomplete CASCADE;
+
+-- Alter project name to text
+ALTER TABLE macrostrat.projects ALTER COLUMN project TYPE TEXT USING project::TEXT;
+
+-- drop custom type
+DROP TYPE IF EXISTS macrostrat.projects_project;
 
-ALTER TABLE macrostrat.projects ADD COLUMN IF NOT EXISTS is_composite BOOLEAN DEFAULT FALSE;
 
+ALTER TABLE macrostrat.projects ADD COLUMN IF NOT EXISTS is_composite BOOLEAN DEFAULT FALSE;
+/** Take the opportunity to add a slug for nicer URLs **/
+ALTER TABLE macrostrat.projects ADD COLUMN IF NOT EXISTS slug TEXT UNIQUE;
 
-CREATE TABLE IF NOT EXISTS macrostrat.project_composite (
+CREATE TABLE IF NOT EXISTS macrostrat.projects_tree (
   id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
-  parent_project_id integer REFERENCES macrostrat.projects(id) ON DELETE CASCADE,
-  child_project_id integer REFERENCES macrostrat.projects(id) ON DELETE CASCADE,
-  UNIQUE (parent_project_id, child_project_id)
+  parent_id integer NOT NULL REFERENCES macrostrat.projects(id) ON DELETE CASCADE ON UPDATE CASCADE,
+  child_id integer NOT NULL REFERENCES macrostrat.projects(id) ON DELETE CASCADE ON UPDATE CASCADE,
+  UNIQUE (parent_id, child_id)
 );
+ALTER TABLE macrostrat.projects_tree OWNER TO macrostrat;
 
 -- Ensure that only composite projects can be parent of other projects
 CREATE OR REPLACE FUNCTION macrostrat.check_composite_parent()
 RETURNS TRIGGER AS $$
 BEGIN
-  IF NOT (SELECT is_composite FROM macrostrat.projects WHERE id = NEW.parent_project_id) THEN
+  IF NOT (SELECT is_composite FROM macrostrat.projects WHERE id = NEW.parent_id) THEN
     RAISE EXCEPTION 'Parent project must be a composite project';
   END IF;
   RETURN NEW;
@@ -22,7 +36,7 @@ $$ LANGUAGE plpgsql;
 
 -- Create the trigger on the project_composite table
 CREATE TRIGGER trg_check_composite_parent
-BEFORE INSERT OR UPDATE ON macrostrat.project_composite
+BEFORE INSERT OR UPDATE ON macrostrat.projects_tree
 FOR EACH ROW EXECUTE FUNCTION macrostrat.check_composite_parent();
 
 /**
@@ -46,28 +60,34 @@ CREATE TRIGGER trg_check_column_project_non_composite
 BEFORE INSERT OR UPDATE ON macrostrat.cols
 FOR EACH ROW EXECUTE FUNCTION macrostrat.check_column_project_non_composite();
 
-
-/** Take the opportunity to add a slug for nicer URLs **/
-ALTER TABLE macrostrat.projects ADD COLUMN IF NOT EXISTS slug TEXT UNIQUE;
-
-/** Function to generate slugs from project names **/
-CREATE OR REPLACE FUNCTION macrostrat.generate_project_slug()
-RETURNS VOID AS $$
+DROP FUNCTION IF EXISTS macrostrat.generate_project_slug(macrostrat.projects);
+CREATE OR REPLACE FUNCTION macrostrat.generate_project_slug(_project macrostrat.projects)
+RETURNS TEXT AS $$
 DECLARE
-  proj RECORD;
   base_slug TEXT;
   unique_slug TEXT;
   suffix INT;
 BEGIN
-  FOR proj IN SELECT id, name FROM macrostrat.projects WHERE slug IS NULL LOOP
-    base_slug := lower(regexp_replace(proj.name, '[^a-zA-Z0-9]+', '-', 'g'));
-    unique_slug := base_slug;
-    suffix := 1;
-    WHILE EXISTS (SELECT 1 FROM macrostrat.projects WHERE slug = unique_slug) LOOP
-      suffix := suffix + 1;
-      unique_slug := base_slug || '-' || suffix;
-    END LOOP;
-    UPDATE macrostrat.projects SET slug = unique_slug WHERE id = proj.id;
+  base_slug := lower(regexp_replace(_project.project, '[^a-zA-Z0-9]+', '-', 'g'));
+  unique_slug := base_slug;
+  suffix := 1;
+  WHILE EXISTS (SELECT 1 FROM macrostrat.projects p WHERE p.slug = unique_slug AND p.id != _project.id) LOOP
+    suffix := suffix + 1;
+    unique_slug := base_slug || '-' || suffix;
   END LOOP;
+  RETURN unique_slug;
 END;
 $$ LANGUAGE plpgsql;
+
+/** Generate slugs from existing projects **/
+UPDATE macrostrat.projects p
+SET slug = macrostrat.generate_project_slug(p)
+WHERE slug IS NULL;
+
+-- Set slug column to NOT NULL
+ALTER TABLE macrostrat.projects ALTER COLUMN slug SET NOT NULL;
+
+
+-- Create an index on the slug column for faster lookups
+CREATE INDEX IF NOT EXISTS idx_projects_slug ON macrostrat.projects(slug);
+
diff --git a/cli/macrostrat/cli/database/migrations/file_storage_updates/__init__.py b/cli/macrostrat/cli/database/migrations/file_storage_updates/__init__.py
@@ -13,7 +13,7 @@ class MapFiles(Migration):
     postconditions = [
         # storage.object no longer has object_group_id
         # intersection table exists in storage schema
-        exists("storage", "map_files"),
+        exists("maps_metadata", "map_files"),
         # intersection table columns exist
         has_columns(
             "maps_metadata",
diff --git a/core/macrostrat/core/migrations/__init__.py b/core/macrostrat/core/migrations/__init__.py
@@ -230,6 +230,9 @@ class Migration:
 
     output_mode: OutputMode = OutputMode.SUMMARY
 
+    def __init__(self):
+        pass
+
     def should_apply(self, database: Database) -> ApplicationStatus:
         """Determine whether this migration can run, or has already run."""
         if self.always_apply:
@@ -509,6 +512,8 @@ def _run_migrations(
 
         # Hack to allow migrations to follow output mode
         _migration.output_mode = output_mode
+
+        print(f"\nApplying migration [bold cyan]{_name}[/]...")
         _migration.apply(db)
         run_counter += 1
         # After running migration, reload the database and confirm that application was sucessful

Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ class BaselineMigration(Migration):`
`12`	`12`	`readiness_state = "ga"`
`13`	`13`	`# Confirm that the tables created by the API v3 migrations are present`
`14`	`14`	`postconditions = [`
`15`		`- exists("storage", "object_group", "object"),`
	`15`	`+ exists("storage", "object"),`
`16`	`16`	`exists("maps_metadata", "ingest_process", "ingest_process_tag"),`
`17`	`17`	`exists("macrostrat_auth", "user", "group"),`
`18`	`18`	`]`