Steps to fix SQL parameters

Author: davenquinn

services/column-footprint-editor/conftest.py

@@ -0,0 +1,10 @@
+import logging
+
+disable_loggers = ["macrostrat.database.utils"]
+
+
+def pytest_configure():
+    # Quiet verbose logging
+    for logger_name in disable_loggers:
+        logger = logging.getLogger(logger_name)
+        logger.disabled = True

services/column-footprint-editor/macrostrat/column_footprint_editor/database/__init__.py

@@ -1,11 +1,12 @@
 from macrostrat.database import Database as BaseDatabase
+from macrostrat.database.utils import get_sql_text
 from pathlib import Path
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
 from .sql_formatter import SqlFormatter
 from ..settings import DATABASE
-from ..utils import config_check, run_docker_config, delete_config
+from ..utils import run_topology_command, delete_config
 
 here = Path(__file__).parent
 fixtures = here / "fixtures"
@@ -35,7 +36,7 @@ def __init__(self, project=None):
 
         self.engine = create_engine(DATABASE, echo=True)
         self.Session = sessionmaker(bind=self.engine)
-        self.config = config_check(project)
+        # self.config = config_check(project)
         self.formatter = SqlFormatter(self.project_id)
 
     def exec_sql(self, sql, params=None, count=None):
@@ -61,17 +62,9 @@ def exec_query(self, filename_or_query, **kwargs):
         """
         from pandas import read_sql
 
-        if "SELECT" in str(filename_or_query):
-            # We are working with a query string instead of
-            # an SQL file.
-            sql = filename_or_query
-            sql = self.formatter.sql_config_format(sql, self.config)
-        else:
-            with open(filename_or_query) as f:
-                sql = f.read()
-            sql = self.formatter.sql_config_format(sql, self.config)
+        txt = get_sql_text(filename_or_query)
 
-        return read_sql(sql, self.engine, **kwargs)
+        return read_sql(txt, self.engine, **kwargs)
 
     #################### db initialization methods ##########################
     def create_project_table(self):
@@ -86,7 +79,7 @@ def clean_topology(self):
 
     def create_project_schema(self):
         # need to run docker command and then create core tables
-        run_docker_config(self.project_id, "create_tables")
+        run_topology_command(self, self.project_id, "create_tables")
         self.run_sql_file(create_core_table)
         self.create_map_face_view()
 
@@ -103,7 +96,7 @@ def insert_project_info(self, params={}):
         self.run_sql_file(project_info_insert, params=params)
 
     def insert_project_column_group(self, params={}):
-        sql = """INSERT INTO ${project_schema}.column_groups(col_group_id, col_group, col_group_name) VALUES(
+        sql = """INSERT INTO {data_schema}.column_groups(col_group_id, col_group, col_group_name) VALUES(
             :col_group_id, :col_group, :col_group_name);"""
         self.run_sql(sql, params)
 
@@ -117,14 +110,14 @@ def redump_linework_from_edge(self):
         self.run_sql_file(redump_linework_sql)
 
     def remove_project(self, params={}):
-        run_docker_config(self.project_id, "delete")  # delete topology
+        run_topology_command(self.project_id, "delete")  # delete topology
         self.run_sql_file(remove_project_schema, params={"project_id": self.project_id})
         delete_config(self.project_id)  # remove config file
 
     ################## db topology methods ##############################
 
     def update_topology(self):
-        run_docker_config(self.project_id, "update")
+        run_topology_command(self.project_id, "update")
 
     ###################### Project-Free methods ########################
 
@@ -142,6 +135,10 @@ def get_next_project_id(self):
         data = self.exec_query(sql).to_dict(orient="records")
         imported_max_id = data[0]["max"]
         all_max_id = data[1]["max"]
+        if imported_max_id is None:
+            imported_max_id = 0
+        if all_max_id is None:
+            all_max_id = 0
         if imported_max_id == all_max_id:
             return imported_max_id + 1000
         else:
@@ -151,9 +148,9 @@ def get_next_col_group_id(self):
         """function to get the next project id that won't conflict with macrostrat"""
         # TODO: unhardcode the max int for project id
         # WARNING: Now this isn't going to be conflict free. Because we split the tables up
-        sql = """SELECT max(col_group_id), 'imported' origin from ${project_schema}.column_groups WHERE col_group_id < 5000
+        sql = """SELECT max(col_group_id), 'imported' origin from {data_schema}.column_groups WHERE col_group_id < 5000
                  UNION ALL
-                 SELECT max(col_group_id), 'all' origin from ${project_schema}.column_groups;"""
+                 SELECT max(col_group_id), 'all' origin from $(data_schema).column_groups;"""
 
         data = self.exec_query(sql).to_dict(orient="records")
         imported_max_id = data[0]["max"]

services/column-footprint-editor/macrostrat/column_footprint_editor/database/fixtures/project_schema_tables.sql

@@ -1,20 +1,20 @@
-CREATE SCHEMA IF NOT EXISTS ${project_schema};
+CREATE SCHEMA IF NOT EXISTS {project_schema};
 
-create table if not exists ${project_schema}.column_groups(
+create table if not exists {project_schema}.column_groups(
     id SERIAL PRIMARY KEY,
     col_group_id integer,
     col_group text,
     col_group_name text,
     color text
 );
 
-CREATE TABLE IF NOT EXISTS ${project_schema}.columns(
+CREATE TABLE IF NOT EXISTS {project_schema}.columns(
     id serial PRIMARY KEY,
     project_id integer,
-    col_group integer REFERENCES ${project_schema}.column_groups(id),
+    col_group integer REFERENCES {project_schema}.column_groups(id),
     col_id integer,
     col_name text,
     description text,
     point geometry,
-    location geometry 
+    location geometry
 );

services/column-footprint-editor/macrostrat/column_footprint_editor/database/fixtures/views.sql

@@ -1,26 +1,26 @@
-ALTER TABLE ${data_schema}.polygon
+ALTER TABLE {data_schema}.polygon
 	ADD COLUMN col_id integer,
-	ADD FOREIGN KEY (col_id) REFERENCES ${project_schema}.columns(id);
+	ADD FOREIGN KEY (col_id) REFERENCES {project_schema}.columns(id);
 
-CREATE OR REPLACE VIEW ${project_schema}.column_map_face AS
+CREATE OR REPLACE VIEW {project_schema}.column_map_face AS
 WITH A as(
 SELECT c.id,
-	   c.project_id, 
-	   c.col_id, 
+	   c.project_id,
+	   c.col_id,
 	   c.col_name,
-	   c.description, 
+	   c.description,
 	   cg.col_group,
 	   cg.id as col_group_id,
 	   cg.col_group_id as col_group_id_macrostrat,
-	   cg.col_group_name, 
+	   cg.col_group_name,
 	   cg.color,
-	   c.point, 
-	   mtm.geometry 
-FROM ${topo_schema}.map_face mtm
-LEFT JOIN ${data_schema}.polygon mdp
+	   c.point,
+	   mtm.geometry
+FROM {topo_schema}.map_face mtm
+LEFT JOIN {data_schema}.polygon mdp
 ON ST_Contains(mtm.geometry, mdp.geometry)
-LEFT JOIN ${project_schema}.columns c
+LEFT JOIN {project_schema}.columns c
 ON mdp.col_id = c.id
-LEFT JOIN ${project_schema}.column_groups cg
+LEFT JOIN {project_schema}.column_groups cg
 ON cg.id = c.col_group
-) SELECT A.* from A;
+) SELECT A.* from A;

services/column-footprint-editor/macrostrat/column_footprint_editor/database/procedures/change-line-coordinates.sql

@@ -1,6 +1,6 @@
 /*
 Updates the existing line, just the geometry
 */
-UPDATE ${data_schema}.linework
+UPDATE {data_schema}.linework
 SET geometry = ST_Multi(ST_GeomFromGeoJSON(:geometry_))
-WHERE id = :id_;
+WHERE id = :id_;

services/column-footprint-editor/macrostrat/column_footprint_editor/database/procedures/clear-project-data.sql

@@ -1,36 +1,36 @@
 SET session_replication_role = replica;
 
-UPDATE ${data_schema}.linework
+UPDATE {data_schema}.linework
 SET
   topo = null,
   geometry_hash = null,
   topology_error = null;
 
-TRUNCATE TABLE ${topo_schema}.face CASCADE;
-TRUNCATE TABLE ${topo_schema}.relation CASCADE;
-TRUNCATE TABLE ${topo_schema}.map_face CASCADE;
+TRUNCATE TABLE {topo_schema}.face CASCADE;
+TRUNCATE TABLE {topo_schema}.relation CASCADE;
+TRUNCATE TABLE {topo_schema}.map_face CASCADE;
 
-INSERT INTO ${topo_schema}.face (face_id) VALUES (0);
+INSERT INTO {topo_schema}.face (face_id) VALUES (0);
 
-ALTER SEQUENCE ${topo_schema}.node_node_id_seq RESTART WITH 1;
-ALTER SEQUENCE ${topo_schema}.face_face_id_seq RESTART WITH 1;
-ALTER SEQUENCE ${topo_schema}.edge_data_edge_id_seq RESTART WITH 1;
-ALTER SEQUENCE ${topo_schema}.topogeo_s_1 RESTART WITH 1;
+ALTER SEQUENCE {topo_schema}.node_node_id_seq RESTART WITH 1;
+ALTER SEQUENCE {topo_schema}.face_face_id_seq RESTART WITH 1;
+ALTER SEQUENCE {topo_schema}.edge_data_edge_id_seq RESTART WITH 1;
+ALTER SEQUENCE {topo_schema}.topogeo_s_1 RESTART WITH 1;
 
-SELECT setval(pg_get_serial_sequence('${topo_schema}.map_face', 'id'), coalesce(max(id),0)+1, false)
-  FROM ${topo_schema}.map_face;
+SELECT setval(pg_get_serial_sequence('{topo_schema}.map_face', 'id'), coalesce(max(id),0)+1, false)
+  FROM {topo_schema}.map_face;
 SET session_replication_role = DEFAULT;
 
-TRUNCATE ${topo_schema}.map_face CASCADE;
+TRUNCATE {topo_schema}.map_face CASCADE;
 
-ALTER SEQUENCE ${topo_schema}.map_face_id_seq RESTART WITH 1;
+ALTER SEQUENCE {topo_schema}.map_face_id_seq RESTART WITH 1;
 
 
-TRUNCATE ${data_schema}.linework CASCADE;
-TRUNCATE ${project_schema}.columns CASCADE;
-TRUNCATE ${data_schema}.polygon CASCADE;
+TRUNCATE {data_schema}.linework CASCADE;
+TRUNCATE {project_schema}.columns CASCADE;
+TRUNCATE {data_schema}.polygon CASCADE;
 
-ALTER SEQUENCE ${data_schema}.linework_id_seq RESTART WITH 1;
-ALTER SEQUENCE ${project_schema}.columns_id_seq RESTART WITH 1;
-ALTER SEQUENCE ${data_schema}.polygon_id_seq RESTART WITH 1;
+ALTER SEQUENCE {data_schema}.linework_id_seq RESTART WITH 1;
+ALTER SEQUENCE {project_schema}.columns_id_seq RESTART WITH 1;
+ALTER SEQUENCE {data_schema}.polygon_id_seq RESTART WITH 1;
 

services/column-footprint-editor/macrostrat/column_footprint_editor/database/procedures/create-line.sql

@@ -1,4 +1,4 @@
-INSERT INTO ${data_schema}.linework(geometry, type) VALUES (
+INSERT INTO {data_schema}.linework(geometry, type) VALUES (
     ST_Multi(ST_GeomFromGeoJSON(:geometry_)),
     'default'
-);
+);

services/column-footprint-editor/macrostrat/column_footprint_editor/database/procedures/delete-line.sql

@@ -1,4 +1,4 @@
 /*
 Deletes a line where the id is equal to the id
 */
-DELETE FROM ${data_schema}.linework WHERE id = :id_;
+DELETE FROM {data_schema}.linework WHERE id = :id_;

services/column-footprint-editor/macrostrat/column_footprint_editor/database/procedures/dump-voronoi-to-lines.sql

@@ -1,7 +1,7 @@
 WITH a as(
     SELECT (ST_Dump(ST_Boundary(ST_GeomFromGeoJSON(:polygon)))).geom as geom
 )
-INSERT INTO ${data_schema}.linework(geometry, type) 
+INSERT INTO {data_schema}.linework(geometry, type)
     SELECT ST_Multi(a.geom),
     'default'
     FROM a;

services/column-footprint-editor/macrostrat/column_footprint_editor/database/procedures/get-bounding-id.sql

@@ -1,11 +1,11 @@
-/* 
+/*
 Select id of polygon that contains point
 */
 
-SELECT 
-    id 
-FROM ${topo_schema}.map_face mtm
+SELECT
+    id
+FROM {topo_schema}.map_face mtm
 WHERE ST_Contains(
-    mtm.geometry, 
+    mtm.geometry,
     ST_GeomFromGeoJSON(:point)
-    );
+    );