Merge pull request #76 from OpenTransitTools/rtp

merge

Author: fpurcell

.gitignore

@@ -15,6 +15,7 @@ COMMIT_EDITMSG
 gtfs.zip
 .DS_Store
 **/tests/*feed
+*.gtfs.zip
 
 # C extensions
 *.so

README.rst

@@ -41,7 +41,9 @@ Install from source via github (if you want the latest code) :
 
    * bin/gtfsdb-load --database_url sqlite:///gtfs.db http://developer.trimet.org/schedule/gtfs.zip
 
-   * bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432 --is_geospatial http://developer.trimet.org/schedule/gtfs.zip
+   * bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432/ott --is_geospatial -s flex gtfsdb/tests/flex-feed.zip
+
+   * bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432/ott --is_geospatial http://developer.trimet.org/schedule/gtfs.zip
 
      .. note:: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqlite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc...
 

gtfsdb/__init__.py

@@ -1,3 +1,5 @@
+#  TODO?  __path__ = __import__('pkgutil').extend_path(__path__, __name__)
+
 from gtfsdb.model.db import Database
 from gtfsdb.model.gtfs import GTFS
 
@@ -6,6 +8,7 @@
 from gtfsdb.model.fare import *  # noqa
 from gtfsdb.model.feed_info import FeedInfo  # noqa
 from gtfsdb.model.frequency import Frequency  # noqa
+from gtfsdb.model.location import *  # noqa
 from gtfsdb.model.route import *  # noqa
 from gtfsdb.model.route_stop import *  # noqa
 from gtfsdb.model.shape import *  # noqa
@@ -39,6 +42,7 @@
     Trip.__name__,
     StopTime.__name__,
     RouteStop.__name__,
+    Location.__name__,
     Frequency.__name__,
     FareAttribute.__name__,
     FareRule.__name__,
@@ -51,4 +55,5 @@
     CurrentRoutes.__name__,
     CurrentRouteStops.__name__,
     CurrentStops.__name__,
-]
+    FlexRegion.__name__,
+]

gtfsdb/config.py

@@ -5,17 +5,25 @@
 
 import os
 import logging.config
-from pkg_resources import resource_filename  # @UnresolvedImport
 
 
+# import pdb; pdb.set_trace()
 """ parse configuration file and setup logging """
 config = ConfigParser()
-ini_file = os.path.join(resource_filename('gtfsdb', 'configs'), 'app.ini')
+ini_file = os.path.join(os.path.dirname(__file__), 'configs', 'app.ini')
 config.read(ini_file)
 if config.has_section('loggers'):
     logging.config.fileConfig(ini_file, disable_existing_loggers=False)
 
 
+def get_value(key, section="general", def_val=None):
+    try:
+        ret_val =config.get(section, key)
+    except:
+        ret_val = def_val
+    return ret_val
+
+
 """ application defaults """
 DEFAULT_BATCH_SIZE = 10000
 DEFAULT_DATABASE_URL = 'sqlite://'
@@ -31,3 +39,9 @@
 
 """ geometry constants """
 SRID = 4326
+
+
+""" misc defaults """
+default_route_color = get_value("default_route_color", def_val="#7B97B2")
+default_frequent_color = get_value("default_frequent_color", def_val="#306095")
+default_text_color = get_value("default_text_color", def_val="#FFFFFF")

gtfsdb/configs/app.ini

@@ -19,3 +19,6 @@ formatter = generic
 [formatter_generic]
 datefmt = %H:%M:%S
 format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+
+[general]
+default_route_color = #7A99B1

gtfsdb/model/agency.py

@@ -12,11 +12,28 @@ class Agency(Base):
     __tablename__ = 'agency'
 
     id = Column(Integer, Sequence(None, optional=True), primary_key=True, nullable=True)
-    agency_id = Column(String(255), index=True, unique=True)
-    agency_name = Column(String(255), nullable=False)
-    agency_url = Column(String(255), nullable=False)
+    agency_id = Column(String(512), index=True, unique=True)
+    agency_name = Column(String(512), nullable=False)
+    agency_url = Column(String(1024), nullable=False)
     agency_timezone = Column(String(50), nullable=False)
     agency_lang = Column(String(10))
     agency_phone = Column(String(50))
-    agency_fare_url = Column(String(255))
-    agency_email = Column(String(255))
+    agency_fare_url = Column(String(1024))
+    agency_email = Column(String(1024))
+    feed_id = Column(String(512))  # optional field that's used in multiple apps, ala OTP
+
+    def feed_agency_id(self):
+        """
+        return an feed_id:agency_id pair
+        """
+        return "{}:{}".format(self.feed_id, self.agency_id)
+
+    @classmethod
+    def post_make_record(cls, row, **kwargs):
+        try:
+            #import pdb; pdb.set_trace()
+            row['feed_id'] = kwargs['feed_id']
+        except:
+            pass
+        return row
+        

gtfsdb/model/base.py

@@ -1,10 +1,9 @@
-from pkg_resources import resource_filename  # @UnresolvedImport
-
 from gtfsdb import config, util
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import object_session
 
 import csv
+import json
 import datetime
 import os
 import sys
@@ -160,38 +159,47 @@ def load(cls, db, **kwargs):
         if cls.datasource == config.DATASOURCE_GTFS:
             directory = kwargs.get('gtfs_directory')
         elif cls.datasource == config.DATASOURCE_LOOKUP:
-            directory = resource_filename('gtfsdb', 'data')
+            directory = util.get_resource_path('data')
 
-        # step 3: load the file
-        log.info("load {0}".format(cls.__name__))
-        records = []
+        # step 3: load the file (.geojson or .csv)
+        log.info("load {}".format(cls.__name__))
         file_path = os.path.join(directory, cls.filename)
         if os.path.exists(file_path):
-            if sys.version_info >= (3, 0):
-                f = open(file_path, 'rb')
-            else:
-                f = open(file_path, 'r')
-            utf8_file = util.UTF8Recoder(f, 'utf-8-sig')
-            reader = csv.DictReader(utf8_file)
-            reader.fieldnames = [field.strip().lower() for field in reader.fieldnames]
             table = cls.__table__
             try:
                 db.engine.execute(table.delete())
             except:
                 log.debug("NOTE: couldn't delete this table")
 
-            i = 0
-            for row in reader:
-                records.append(cls.make_record(row))
-                i += 1
-                if i >= batch_size:
-                    db.engine.execute(table.insert(), records)
-                    sys.stdout.write('*')
-                    records = []
-                    i = 0
-            if len(records) > 0:
-                db.engine.execute(table.insert(), records)
-            f.close()
+            if 'geojson' in cls.filename:
+                log.debug('{}.load importing geojson data from {}'.format(cls.__name__, cls.filename))
+                #import pdb; pdb.set_trace()
+                with open(file_path) as f:
+                    data = json.load(f)
+                    for row in data['features']:
+                        rec = cls.make_record(row, **kwargs)
+                        db.engine.execute(table.insert(), rec)                    
+            else:
+                if sys.version_info >= (3, 0):
+                    f = open(file_path, 'rb')
+                else:
+                    f = open(file_path, 'r')
+                utf8_file = util.UTF8Recoder(f, 'utf-8-sig')
+                reader = csv.DictReader(utf8_file)
+                reader.fieldnames = [field.strip().lower() for field in reader.fieldnames]
+                records = []
+                i = 0
+                for row in reader:
+                    records.append(cls.make_record(row, **kwargs))
+                    i += 1
+                    if i >= batch_size:
+                        util.safe_db_engine_load(db, table, records)
+                        sys.stdout.write('*')
+                        records = []
+                        i = 0
+                if len(records) > 0:
+                    util.safe_db_engine_load(db, table, records)
+                f.close()
 
         # step 4: done...
         process_time = time.time() - start_time
@@ -207,7 +215,7 @@ def post_process(cls, db, **kwargs):
         pass
 
     @classmethod
-    def make_record(cls, row):
+    def make_record(cls, row, **kwargs):
         for k, v in row.copy().items():
             if isinstance(v, basestring):
                 row[k] = v.strip()
@@ -230,11 +238,11 @@ def make_record(cls, row):
             cls.add_geom_to_dict(row)
 
         """ post make_record gives the calling class a chance to fix things up prior to being sent down to database """
-        row = cls.post_make_record(row)
+        row = cls.post_make_record(row, **kwargs)
         return row
 
     @classmethod
-    def post_make_record(cls, row):
+    def post_make_record(cls, row, **kwargs):
         """ Base does nothing, but a derived class now has a chance to clean up the record prior to db commit """
         return row
 

gtfsdb/model/block.py

@@ -27,13 +27,13 @@ class Block(Base):
 
     id = Column(Integer, Sequence(None, optional=True), primary_key=True)
     sequence = Column(Integer)
-    block_id = Column(String(255), index=True, nullable=False)
-    service_id = Column(String(255), index=True, nullable=False)
-    trip_id = Column(String(255), index=True, nullable=False)
-    prev_trip_id = Column(String(255))
-    next_trip_id = Column(String(255))
-    start_stop_id = Column(String(255), index=True, nullable=False)
-    end_stop_id = Column(String(255), index=True, nullable=False)
+    block_id = Column(String(512), index=True, nullable=False)
+    service_id = Column(String(512), index=True, nullable=False)
+    trip_id = Column(String(512), index=True, nullable=False)
+    prev_trip_id = Column(String(512))
+    next_trip_id = Column(String(512))
+    start_stop_id = Column(String(512), index=True, nullable=False)
+    end_stop_id = Column(String(512), index=True, nullable=False)
 
     universal_calendar = relationship(
         'UniversalCalendar',

gtfsdb/model/calendar.py

@@ -24,7 +24,7 @@ class Calendar(Base):
     __tablename__ = 'calendar'
     __table_args__ = (Index('calendar_ix1', 'start_date', 'end_date'),)
 
-    service_id = Column(String(255), primary_key=True, index=True, nullable=False)
+    service_id = Column(String(512), primary_key=True, index=True, nullable=False)
     monday = Column(SmallInteger, nullable=False)
     tuesday = Column(SmallInteger, nullable=False)
     wednesday = Column(SmallInteger, nullable=False)
@@ -34,7 +34,7 @@ class Calendar(Base):
     sunday = Column(SmallInteger, nullable=False)
     start_date = Column(Date, nullable=False)
     end_date = Column(Date, nullable=False)
-    service_name = Column(String(255))  # Trillium extension, a human-readable name for the calendar.
+    service_name = Column(String(512))  # Trillium extension, a human-readable name for the calendar.
 
     def weekday_list(self):
         weekday_dict = dict(monday=0, tuesday=1, wednesday=2, thursday=3, friday=4, saturday=5, sunday=6)
@@ -62,7 +62,7 @@ class CalendarDate(Base):
 
     __tablename__ = 'calendar_dates'
 
-    service_id = Column(String(255), primary_key=True, index=True, nullable=False)
+    service_id = Column(String(512), primary_key=True, index=True, nullable=False)
     date = Column(Date, primary_key=True, index=True, nullable=False)
     exception_type = Column(Integer, nullable=False)
 
@@ -79,7 +79,7 @@ class UniversalCalendar(Base):
     datasource = config.DATASOURCE_DERIVED
     __tablename__ = 'universal_calendar'
 
-    service_id = Column(String(255), primary_key=True, index=True, nullable=False)
+    service_id = Column(String(512), primary_key=True, index=True, nullable=False)
     date = Column(Date, primary_key=True, index=True, nullable=False)
 
     trips = relationship(
@@ -97,12 +97,26 @@ def load(cls, db, **kwargs):
         session.commit()
         q = session.query(CalendarDate)
         for calendar_date in q:
-            cd_kwargs = dict(date=calendar_date.date,
-                             service_id=calendar_date.service_id)
+            cd_kwargs = dict(date=calendar_date.date, service_id=calendar_date.service_id)
             if calendar_date.is_addition:
                 session.merge(cls(**cd_kwargs))
             if calendar_date.is_removal:
-                session.query(cls).filter_by(**cd_kwargs).delete()
+                try:
+                    # import pdb; pdb.set_trace()
+                    """
+                        TODO 11-1-2023 - exceptions thrown by executing this delete indicate that the ORM might be in a very funky state ... shows that former agency data is in ORM
+
+                        2023-10-30 12:21:23,341 - gtfsdb.model.calendar (line 109) - WARNING - (psycopg2.errors.UndefinedTable) invalid reference to FROM-clause entry for table "universal_calendar"
+                        LINE 1: DELETE FROM cat.universal_calendar WHERE canby.universal_cal...
+                        HINT:  There is an entry for table "universal_calendar", but it cannot be referenced from this part of the query.
+                        [SQL: DELETE FROM cat.universal_calendar WHERE canby.universal_calendar.date = %(date_1)s AND canby.universal_calendar.service_id = %(service_id_1)s]
+                        [parameters: {'date_1': datetime.date(2024, 1, 1), 'service_id_1': 'c_69855_b_81444_d_31'}]
+                    """
+                    session.query(cls).filter_by(**cd_kwargs).delete()
+                except Exception as e:
+                    log.warning(e)
+
         session.commit()
+        session.flush()
         process_time = time.time() - start_time
         log.debug('{0}.load ({1:.0f} seconds)'.format(cls.__name__, process_time))

gtfsdb/model/db.py

@@ -24,12 +24,13 @@ def __init__(self, **kwargs):
             url = kwargs.get('database_url', config.DEFAULT_DATABASE_URL)
         self.url = url
         self.schema = kwargs.get('schema', config.DEFAULT_SCHEMA)
+        self.feed_id = kwargs.get('feed_id', None)
         self.is_geospatial = kwargs.get('is_geospatial', config.DEFAULT_IS_GEOSPATIAL)
 
         """Order list of class names, used for creating & populating tables"""
         from gtfsdb import SORTED_CLASS_NAMES, CURRENT_CLASS_NAMES
         self.sorted_class_names = SORTED_CLASS_NAMES
-        if kwargs.get('current_tables'):
+        if kwargs.get('current_tables') or kwargs.get('current_tables_all'):
             self.sorted_class_names.extend(CURRENT_CLASS_NAMES)
         # import pdb; pdb.set_trace()