sqlalchemy upgrade version adjustments

Author: atalyaalon

alembic/versions/312c9eb92e40_add_cbs_locations_table.py

@@ -34,7 +34,7 @@ def upgrade():
     sa.PrimaryKeyConstraint('id')
     )
     conn = op.get_bind()
-    conn.execute("""INSERT INTO cbs_locations 
+    conn.execute(sa.text("""INSERT INTO cbs_locations 
                         (SELECT ROW_NUMBER() OVER (ORDER BY road1) as id, LOCATIONS.*
                         FROM 
                         (SELECT DISTINCT road1,
@@ -52,7 +52,7 @@ def upgrade():
                         WHERE (provider_code=1
                                OR provider_code=3)
                           AND (longitude is not null
-                               AND latitude is not null)) LOCATIONS)""")
+                               AND latitude is not null)) LOCATIONS)"""))
     # ### end Alembic commands ###
 
 

alembic/versions/4c4b79f8c4a_adding_geom_gix_to_markers.py

@@ -12,20 +12,21 @@
 depends_on = None
 
 from alembic import op
+import sqlalchemy as sa
 
 
 def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
     conn = op.get_bind()
-    conn.execute('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));')
-    conn.execute('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));')
+    conn.execute(sa.text('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));'))
+    conn.execute(sa.text('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));'))
 
     ### end Alembic commands ###
 
 
 def downgrade():
     ### commands auto generated by Alembic - please adjust! ###
     conn = op.get_bind()
-    conn.execute('DROP INDEX geom_gix;')
-    conn.execute('DROP INDEX discussions_gix;')
+    conn.execute(sa.text('DROP INDEX geom_gix;'))
+    conn.execute(sa.text('DROP INDEX discussions_gix;'))
     ### end Alembic commands ###

alembic/versions/5a5ffe56bb7_adding_geom_table_to_markers_and_discussions.py

@@ -12,31 +12,32 @@
 depends_on = None
 
 from alembic import op
+import sqlalchemy as sa
 
 
 def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
     conn = op.get_bind()
-    conn.execute('CREATE EXTENSION IF NOT EXISTS postgis;')
-    conn.execute('CREATE EXTENSION IF NOT EXISTS postgis_topology;')
-    conn.execute("SELECT AddGeometryColumn('public','markers','geom',4326,'POINT',2);")
-    conn.execute('UPDATE markers SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);')
-    conn.execute('CREATE INDEX idx_markers_geom ON markers USING GIST(geom);')
-    conn.execute("SELECT AddGeometryColumn('public','discussions','geom',4326,'POINT',2);")
-    conn.execute('UPDATE discussions SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);')
-    conn.execute('CREATE INDEX idx_discussions_geom ON discussions USING GIST(geom);')
+    conn.execute(sa.text('CREATE EXTENSION IF NOT EXISTS postgis;'))
+    conn.execute(sa.text('CREATE EXTENSION IF NOT EXISTS postgis_topology;'))
+    conn.execute(sa.text("SELECT AddGeometryColumn('public','markers','geom',4326,'POINT',2);"))
+    conn.execute(sa.text('UPDATE markers SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);'))
+    conn.execute(sa.text('CREATE INDEX idx_markers_geom ON markers USING GIST(geom);'))
+    conn.execute(sa.text("SELECT AddGeometryColumn('public','discussions','geom',4326,'POINT',2);"))
+    conn.execute(sa.text('UPDATE discussions SET geom = ST_SetSRID(ST_MakePoint(longitude,latitude),4326);'))
+    conn.execute(sa.text('CREATE INDEX idx_discussions_geom ON discussions USING GIST(geom);'))
 
     ### end Alembic commands ###
 
 
 def downgrade():
     ### commands auto generated by Alembic - please adjust! ###
     conn = op.get_bind()
-    conn.execute('DROP INDEX idx_markers_geom;')
+    conn.execute(sa.text('DROP INDEX idx_markers_geom;'))
     op.drop_column('markers', 'geom')
-    conn.execute('DROP INDEX idx_discussions_geom;')
+    conn.execute(sa.text('DROP INDEX idx_discussions_geom;'))
     op.drop_column('discussions', 'geom')
-    conn.execute('DROP EXTENSION postgis_topology;')
-    conn.execute('DROP EXTENSION postgis;')
-    conn.execute('DROP SCHEMA IF EXISTS topology CASCADE;')
+    conn.execute(sa.text('DROP EXTENSION postgis_topology;'))
+    conn.execute(sa.text('DROP EXTENSION postgis;'))
+    conn.execute(sa.text('DROP SCHEMA IF EXISTS topology CASCADE;'))
     ### end Alembic commands ###

alembic/versions/7574885e1fed_remove_unecessary_table_index.py

@@ -34,6 +34,6 @@ def downgrade():
     op.create_index('provider_and_id_idx_involved', 'involved', ['provider_and_id'], unique=False)
     op.create_index('provider_and_id_idx_vehicles', 'vehicles', ['provider_and_id'], unique=False)
     conn = op.get_bind()
-    conn.execute('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));')
-    conn.execute('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));')
+    conn.execute(sa.text('CREATE INDEX geom_gix ON markers USING GIST (geography(geom));'))
+    conn.execute(sa.text('CREATE INDEX discussions_gix ON discussions USING GIST (geography(geom));'))
     # ### end Alembic commands ###

alembic/versions/7f629b4c8891_add_news_flash_fields.py

@@ -19,9 +19,9 @@
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     conn = op.get_bind()
-    conn.execute("ALTER TABLE news_flash ALTER COLUMN id SET DEFAULT nextval('news_flash_id_seq');")
-    conn.execute("ALTER SEQUENCE news_flash_id_seq OWNED BY news_flash.id;")
-    conn.execute("SELECT setval('news_flash_id_seq', COALESCE(max(id), 1)) FROM news_flash;")
+    conn.execute(sa.text("ALTER TABLE news_flash ALTER COLUMN id SET DEFAULT nextval('news_flash_id_seq');"))
+    conn.execute(sa.text("ALTER SEQUENCE news_flash_id_seq OWNED BY news_flash.id;"))
+    conn.execute(sa.text("SELECT setval('news_flash_id_seq', COALESCE(max(id), 1)) FROM news_flash;"))
     op.add_column('news_flash', sa.Column('district_hebrew', sa.Text(), nullable=True))
     op.add_column('news_flash', sa.Column('non_urban_intersection_hebrew', sa.Text(), nullable=True))
     op.add_column('news_flash', sa.Column('region_hebrew', sa.Text(), nullable=True))

anyway/parsers/infographics_data_cache_updater.py

@@ -2,6 +2,7 @@
 
 from datetime import datetime
 from sqlalchemy import not_
+import sqlalchemy as sa
 from anyway.models import (
     Base,
     InfographicsDataCache,
@@ -214,13 +215,15 @@ def copy_temp_into_cache(table: Dict[str, Base]):
     db.session.commit()
     start = datetime.now()
     with db.get_engine().begin() as conn:
-        conn.execute("lock table infographics_data_cache in exclusive mode")
+        conn.execute(sa.text("lock table infographics_data_cache in exclusive mode"))
         logging.debug(f"in transaction, after lock")
-        conn.execute(f"delete from {table[CACHE].__tablename__}")
+        conn.execute(sa.text(f"delete from {table[CACHE].__tablename__}"))
         logging.debug(f"in transaction, after delete")
         conn.execute(
-            f"insert into {table[CACHE].__tablename__} "
-            f"SELECT * from {table[TEMP].__tablename__}"
+            sa.text(
+                f"insert into {table[CACHE].__tablename__} "
+                f"SELECT * from {table[TEMP].__tablename__}"
+            )
         )
         logging.debug(f"in transaction, after insert into")
     logging.info(f"cache unavailable time: {str(datetime.now() - start)}")

anyway/parsers/registered.py

@@ -7,7 +7,7 @@
 from datetime import datetime
 from anyway.models import RegisteredVehicle, DeprecatedCity
 from anyway.utilities import time_delta, CsvReader, ImporterUI, truncate_tables, decode_hebrew
-from anyway.app_and_db import db
+from anyway.app_and_db import db, app
 
 
 COLUMN_CITY_NAME_ENG = 0
@@ -71,8 +71,8 @@ def import_file(self, inputfile):
             else:
                 self.header_row(row)
             row_count += 1
-
-        db.session.bulk_insert_mappings(RegisteredVehicle, inserts)
+        with app.app_context():
+            db.session.bulk_insert_mappings(RegisteredVehicle, inserts)
         return total
 
     @staticmethod
@@ -131,11 +131,11 @@ def main(specific_folder, delete_all, path):
     started = datetime.now()
     for fname in dir_files:
         total += importer.import_file(fname)
-
-    db.session.commit()
-    db.engine.execute(
-        "UPDATE {0} SET city_id = (SELECT id FROM {1} WHERE {0}.search_name = {1}.search_heb) WHERE city_id IS NULL".format(
-            RegisteredVehicle.__tablename__, DeprecatedCity.__tablename__
+    with app.app_context():
+        db.session.commit()
+        db.engine.execute(
+            "UPDATE {0} SET city_id = (SELECT id FROM {1} WHERE {0}.search_name = {1}.search_heb) WHERE city_id IS NULL".format(
+                RegisteredVehicle.__tablename__, DeprecatedCity.__tablename__
+            )
         )
-    )
-    logging.info("Total: {0} items in {1}".format(total, time_delta(started)))
+        logging.info("Total: {0} items in {1}".format(total, time_delta(started)))

anyway/utilities.py

@@ -12,6 +12,7 @@
 from urllib.parse import urlparse
 from sqlalchemy import func, or_
 from sqlalchemy.sql import select
+import sqlalchemy as sa
 
 import phonenumbers
 from dateutil.relativedelta import relativedelta
@@ -196,7 +197,7 @@ def truncate_tables(db, tables):
 def delete_all_rows_from_table(conn, table):
     table_name = table.__tablename__
     logging.info("Deleting all rows from table " + table_name)
-    conn.execute("DELETE FROM " + table_name)
+    conn.execute(sa.text("DELETE FROM " + table_name))
 
 
 def split_query_to_chunks_by_column(base_select, column_to_chunk_by, chunk_size, conn):