Skip to content

Modernize rebuild scripts #231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
davenquinn wants to merge 10 commits into
base: main
Choose a base branch
from
Draft

Modernize rebuild scripts #231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
30a47a7
Update autocomplete script for PostgreSQL
davenquinn Dec 17, 2025
863c144
Start working on lookup_strat_names
davenquinn Dec 17, 2025
6deba42
Updated lookup_strat_names script, not quite finished
davenquinn Dec 17, 2025
a596910
Update lookup-strat-names script
davenquinn Dec 17, 2025
39b3a38
Partially working lookup-strat-names script
davenquinn Dec 17, 2025
cf5d25c
Format code and sort imports
davenquinn Dec 17, 2025
e7232ed
Reverted control flow error
davenquinn Dec 17, 2025
478e917
Merge remote-tracking branch 'origin/modernize-rebuild-scripts' into …
davenquinn Dec 17, 2025
83fecc3
Small control flow improvements
davenquinn Dec 17, 2025
1756b84
Format code and sort imports
davenquinn Dec 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .idea/misc.xml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion cli/macrostrat/cli/commands/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from psycopg2.extras import NamedTupleCursor

from ..database import mariaConnection, pgConnection
from ..database._legacy import pgConnection


class Base(object):
Expand Down
26 changes: 12 additions & 14 deletions cli/macrostrat/cli/commands/process_scripts/tesselate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,12 @@
import numpy as np
import pyproj
import shapely.ops as ops
from psycopg2.extensions import AsIs
from psycopg2.extras import NamedTupleCursor
from scipy.spatial import Voronoi
from shapely.geometry import MultiPoint, Point, Polygon, mapping, shape
from shapely.ops import cascaded_union
from shapely.wkt import loads

from .. import schlep
# TODO: don't need this any longer
# from .. import schlep
from ..base import Base


Expand Down Expand Up @@ -660,16 +658,16 @@ def run(self, args):
self.mariadb["connection"].close()

# Update postgres
schlep_instance = schlep(
{
"pg": self.pg["raw_connection"],
"mariadb": self.mariadb["raw_connection"],
},
[None, ""],
)

schlep_instance.move_table("col_areas")
schlep_instance.move_table("cols")
# schlep_instance = schlep(
# {
# "pg": self.pg["raw_connection"],
# "mariadb": self.mariadb["raw_connection"],
# },
# [None, ""],
# )
#
# schlep_instance.move_table("col_areas")
# schlep_instance.move_table("cols")

urllib.request.urlopen(
"http://127.0.0.1:5000/api/v2/columns/refresh-cache?cacheRefreshKey=%s"
Expand Down
176 changes: 10 additions & 166 deletions cli/macrostrat/cli/commands/rebuild_scripts/autocomplete.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,174 +1,18 @@
from pathlib import Path

from ...database import get_db
from ..base import Base

here = Path(__file__).parent

# NOTE: this was successfully migrated from MariaDB to PostgreSQL on 2025-12-16


class Autocomplete(Base):
def __init__(self, *args):
Base.__init__(self, {}, *args)

def run(self):
self.mariadb["cursor"].execute(
"""
DROP TABLE IF EXISTS autocomplete_new;
"""
)
self.mariadb["connection"].commit()
self.mariadb["cursor"].execute(
"""
DROP TABLE IF EXISTS autocomplete_old;
"""
)
self.mariadb["connection"].commit()

# Build the new table
self.mariadb["cursor"].execute(
"""
CREATE TABLE autocomplete_new AS
SELECT * FROM (
select id, econ as name, 'econs' as type, 'econ' as category from econs
union
select 0 AS id, econ_type AS name, 'econ_types' AS type, 'econ' as category
FROM econs
WHERE econ != econ_type
GROUP BY econ_type
union
SELECT 0 AS id, econ_class AS name, 'econ_classes' AS type, 'econ' as category FROM econs GROUP BY econ_class
union
select id, environ as name, 'environments' as type, 'environ' as category
FROM environs
WHERE environ != environ_class
union
select 0 AS id, environ_type AS name, 'environment_types' AS type, 'environ' as category
FROM environs
GROUP BY environ_type
union
select 0 AS id, environ_class AS name, 'environment_classes' AS type, 'environ' as category
FROM environs
GROUP BY environ_class
union
select id, concat(lith_att, ' (', att_type, ')') as name, 'lithology_attributes' as type, 'lith_att' as category from lith_atts
union
select id, project as name, 'projects' as type, 'project' as category from projects
union
SELECT DISTINCT strat_names_meta.concept_id AS id, name, 'strat_name_concepts' AS type, 'strat_name' as category
FROM strat_names_meta
JOIN strat_names ON strat_names_meta.concept_id = strat_names.concept_id
union
(select id, CONCAT(strat_name, ' ', rank) AS name, 'strat_name_orphans' as type, 'strat_name' as category from strat_names WHERE concept_id = 0)
union
select id, col_name as name, 'columns' as type, 'column' as category from cols
union
select id, col_group_long as name, 'groups' as type, 'group' as category from col_groups
union
SELECT id, lith AS name, 'lithologies' AS type, 'lithology' as category
FROM liths
WHERE lith != lith_type AND lith != lith_class
union
SELECT 0 AS id, lith_type AS name, 'lithology_types' AS type, 'lithology' AS category
FROM liths
WHERE lith_type != lith_class
GROUP BY lith_type
union
SELECT 0 AS id, lith_class AS name, 'lithology_classes' AS type, 'lithology' as category
FROM liths
GROUP BY lith_class
union
select id, interval_name as name, 'intervals' as type, 'interval' as category from intervals
union
SELECT id, mineral AS name, 'minerals' AS type, 'mineral' as category from minerals
union
SELECT id, structure as name, 'structures' as type, 'structure' as category from structures
) i;
"""
)
self.mariadb["cursor"].close()
self.mariadb["cursor"] = self.mariadb["connection"].cursor()

self.mariadb["cursor"].execute(
"""
UPDATE autocomplete_new AS a
INNER JOIN (
SELECT concept_id, CONCAT(name, COALESCE(CONCAT(' (', interval_name, ')'), '')) AS name
FROM strat_names_meta
LEFT JOIN intervals ON intervals.id = strat_names_meta.interval_id
) sub ON a.id = sub.concept_id
SET a.name = sub.name
WHERE a.id IN (
SELECT id FROM (
SELECT id
FROM autocomplete_new
WHERE name IN (
SELECT name
FROM (
SELECT name, type, count(*)
FROM autocomplete_new
WHERE type = 'strat_name_concepts'
GROUP BY name, type
HAVING count(*) > 1
ORDER BY count(*) desc
) a
)
) b
) AND type = 'strat_name_concepts';
"""
)
self.mariadb["cursor"].close()
self.mariadb["cursor"] = self.mariadb["connection"].cursor()

self.mariadb["cursor"].execute(
"""
UPDATE autocomplete_new AS a
INNER JOIN (
SELECT DISTINCT strat_names.id, CONCAT(strat_name, ' (', FO_period, ')') AS name
FROM strat_names
JOIN unit_strat_names ON strat_names.id = unit_strat_names.strat_name_id
JOIN lookup_unit_intervals ON lookup_unit_intervals.unit_id = unit_strat_names.unit_id
) sub ON a.id = sub.id
SET a.name = sub.name
WHERE a.id IN (
SELECT id FROM (
SELECT id
FROM autocomplete_new
WHERE name IN (
SELECT name
FROM (
SELECT name, type, count(*)
FROM autocomplete_new
WHERE type = 'strat_name_orphans'
GROUP BY name, type
HAVING count(*) > 1
ORDER BY count(*) desc
) a
)
) b
) AND type = 'strat_name_orphans';
"""
)
self.mariadb["cursor"].close()
self.mariadb["cursor"] = self.mariadb["connection"].cursor()

self.mariadb["cursor"].execute(
"""
ALTER TABLE autocomplete rename to autocomplete_old;
"""
)
self.mariadb["cursor"].close()
self.mariadb["cursor"] = self.mariadb["connection"].cursor()

self.mariadb["cursor"].execute(
"""
ALTER TABLE autocomplete_new rename to autocomplete;
"""
)
self.mariadb["cursor"].close()
self.mariadb["cursor"] = self.mariadb["connection"].cursor()

self.mariadb["cursor"].execute(
"""
DROP TABLE IF EXISTS autocomplete_old;
"""
)
self.mariadb["cursor"].close()
self.mariadb["cursor"] = self.mariadb["connection"].cursor()

self.mariadb["cursor"].close()
self.mariadb["connection"].close()
db = get_db()
db.run_sql(here / "sql" / "autocomplete.sql")
# TODO: synchonize macrostrat_api.autocomplete view
Loading