Skip to content

Commit 9fb3852

Browse files
authored
Merge pull request #76 from OpenTransitTools/rtp
merge
2 parents 38c20e1 + 6096d86 commit 9fb3852

36 files changed

+758
-257
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ COMMIT_EDITMSG
1515
gtfs.zip
1616
.DS_Store
1717
**/tests/*feed
18+
*.gtfs.zip
1819

1920
# C extensions
2021
*.so

README.rst

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ Install from source via github (if you want the latest code) :
4141

4242
* bin/gtfsdb-load --database_url sqlite:///gtfs.db http://developer.trimet.org/schedule/gtfs.zip
4343

44-
* bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432 --is_geospatial http://developer.trimet.org/schedule/gtfs.zip
44+
* bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432/ott --is_geospatial -s flex gtfsdb/tests/flex-feed.zip
45+
46+
* bin/gtfsdb-load --database_url postgresql://postgres@localhost:5432/ott --is_geospatial http://developer.trimet.org/schedule/gtfs.zip
4547

4648
.. note:: adding the `is_geospatial` cmdline flag, when paired with a spatial-database ala PostGIS (e.g., is_spatial is meaningless with sqlite), will take longer to load...but will create geometry columns for both rendering and calculating nearest distances, etc...
4749

gtfsdb/__init__.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# TODO? __path__ = __import__('pkgutil').extend_path(__path__, __name__)
2+
13
from gtfsdb.model.db import Database
24
from gtfsdb.model.gtfs import GTFS
35

@@ -6,6 +8,7 @@
68
from gtfsdb.model.fare import * # noqa
79
from gtfsdb.model.feed_info import FeedInfo # noqa
810
from gtfsdb.model.frequency import Frequency # noqa
11+
from gtfsdb.model.location import * # noqa
912
from gtfsdb.model.route import * # noqa
1013
from gtfsdb.model.route_stop import * # noqa
1114
from gtfsdb.model.shape import * # noqa
@@ -39,6 +42,7 @@
3942
Trip.__name__,
4043
StopTime.__name__,
4144
RouteStop.__name__,
45+
Location.__name__,
4246
Frequency.__name__,
4347
FareAttribute.__name__,
4448
FareRule.__name__,
@@ -51,4 +55,5 @@
5155
CurrentRoutes.__name__,
5256
CurrentRouteStops.__name__,
5357
CurrentStops.__name__,
54-
]
58+
FlexRegion.__name__,
59+
]

gtfsdb/config.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,25 @@
55

66
import os
77
import logging.config
8-
from pkg_resources import resource_filename # @UnresolvedImport
98

109

10+
# import pdb; pdb.set_trace()
1111
""" parse configuration file and setup logging """
1212
config = ConfigParser()
13-
ini_file = os.path.join(resource_filename('gtfsdb', 'configs'), 'app.ini')
13+
ini_file = os.path.join(os.path.dirname(__file__), 'configs', 'app.ini')
1414
config.read(ini_file)
1515
if config.has_section('loggers'):
1616
logging.config.fileConfig(ini_file, disable_existing_loggers=False)
1717

1818

19+
def get_value(key, section="general", def_val=None):
20+
try:
21+
ret_val =config.get(section, key)
22+
except:
23+
ret_val = def_val
24+
return ret_val
25+
26+
1927
""" application defaults """
2028
DEFAULT_BATCH_SIZE = 10000
2129
DEFAULT_DATABASE_URL = 'sqlite://'
@@ -31,3 +39,9 @@
3139

3240
""" geometry constants """
3341
SRID = 4326
42+
43+
44+
""" misc defaults """
45+
default_route_color = get_value("default_route_color", def_val="#7B97B2")
46+
default_frequent_color = get_value("default_frequent_color", def_val="#306095")
47+
default_text_color = get_value("default_text_color", def_val="#FFFFFF")

gtfsdb/configs/app.ini

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,6 @@ formatter = generic
1919
[formatter_generic]
2020
datefmt = %H:%M:%S
2121
format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
22+
23+
[general]
24+
default_route_color = #7A99B1

gtfsdb/model/agency.py

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,28 @@ class Agency(Base):
1212
__tablename__ = 'agency'
1313

1414
id = Column(Integer, Sequence(None, optional=True), primary_key=True, nullable=True)
15-
agency_id = Column(String(255), index=True, unique=True)
16-
agency_name = Column(String(255), nullable=False)
17-
agency_url = Column(String(255), nullable=False)
15+
agency_id = Column(String(512), index=True, unique=True)
16+
agency_name = Column(String(512), nullable=False)
17+
agency_url = Column(String(1024), nullable=False)
1818
agency_timezone = Column(String(50), nullable=False)
1919
agency_lang = Column(String(10))
2020
agency_phone = Column(String(50))
21-
agency_fare_url = Column(String(255))
22-
agency_email = Column(String(255))
21+
agency_fare_url = Column(String(1024))
22+
agency_email = Column(String(1024))
23+
feed_id = Column(String(512)) # optional field that's used in multiple apps, ala OTP
24+
25+
def feed_agency_id(self):
26+
"""
27+
return an feed_id:agency_id pair
28+
"""
29+
return "{}:{}".format(self.feed_id, self.agency_id)
30+
31+
@classmethod
32+
def post_make_record(cls, row, **kwargs):
33+
try:
34+
#import pdb; pdb.set_trace()
35+
row['feed_id'] = kwargs['feed_id']
36+
except:
37+
pass
38+
return row
39+

gtfsdb/model/base.py

Lines changed: 36 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
from pkg_resources import resource_filename # @UnresolvedImport
2-
31
from gtfsdb import config, util
42
from sqlalchemy.ext.declarative import declarative_base
53
from sqlalchemy.orm import object_session
64

75
import csv
6+
import json
87
import datetime
98
import os
109
import sys
@@ -160,38 +159,47 @@ def load(cls, db, **kwargs):
160159
if cls.datasource == config.DATASOURCE_GTFS:
161160
directory = kwargs.get('gtfs_directory')
162161
elif cls.datasource == config.DATASOURCE_LOOKUP:
163-
directory = resource_filename('gtfsdb', 'data')
162+
directory = util.get_resource_path('data')
164163

165-
# step 3: load the file
166-
log.info("load {0}".format(cls.__name__))
167-
records = []
164+
# step 3: load the file (.geojson or .csv)
165+
log.info("load {}".format(cls.__name__))
168166
file_path = os.path.join(directory, cls.filename)
169167
if os.path.exists(file_path):
170-
if sys.version_info >= (3, 0):
171-
f = open(file_path, 'rb')
172-
else:
173-
f = open(file_path, 'r')
174-
utf8_file = util.UTF8Recoder(f, 'utf-8-sig')
175-
reader = csv.DictReader(utf8_file)
176-
reader.fieldnames = [field.strip().lower() for field in reader.fieldnames]
177168
table = cls.__table__
178169
try:
179170
db.engine.execute(table.delete())
180171
except:
181172
log.debug("NOTE: couldn't delete this table")
182173

183-
i = 0
184-
for row in reader:
185-
records.append(cls.make_record(row))
186-
i += 1
187-
if i >= batch_size:
188-
db.engine.execute(table.insert(), records)
189-
sys.stdout.write('*')
190-
records = []
191-
i = 0
192-
if len(records) > 0:
193-
db.engine.execute(table.insert(), records)
194-
f.close()
174+
if 'geojson' in cls.filename:
175+
log.debug('{}.load importing geojson data from {}'.format(cls.__name__, cls.filename))
176+
#import pdb; pdb.set_trace()
177+
with open(file_path) as f:
178+
data = json.load(f)
179+
for row in data['features']:
180+
rec = cls.make_record(row, **kwargs)
181+
db.engine.execute(table.insert(), rec)
182+
else:
183+
if sys.version_info >= (3, 0):
184+
f = open(file_path, 'rb')
185+
else:
186+
f = open(file_path, 'r')
187+
utf8_file = util.UTF8Recoder(f, 'utf-8-sig')
188+
reader = csv.DictReader(utf8_file)
189+
reader.fieldnames = [field.strip().lower() for field in reader.fieldnames]
190+
records = []
191+
i = 0
192+
for row in reader:
193+
records.append(cls.make_record(row, **kwargs))
194+
i += 1
195+
if i >= batch_size:
196+
util.safe_db_engine_load(db, table, records)
197+
sys.stdout.write('*')
198+
records = []
199+
i = 0
200+
if len(records) > 0:
201+
util.safe_db_engine_load(db, table, records)
202+
f.close()
195203

196204
# step 4: done...
197205
process_time = time.time() - start_time
@@ -207,7 +215,7 @@ def post_process(cls, db, **kwargs):
207215
pass
208216

209217
@classmethod
210-
def make_record(cls, row):
218+
def make_record(cls, row, **kwargs):
211219
for k, v in row.copy().items():
212220
if isinstance(v, basestring):
213221
row[k] = v.strip()
@@ -230,11 +238,11 @@ def make_record(cls, row):
230238
cls.add_geom_to_dict(row)
231239

232240
""" post make_record gives the calling class a chance to fix things up prior to being sent down to database """
233-
row = cls.post_make_record(row)
241+
row = cls.post_make_record(row, **kwargs)
234242
return row
235243

236244
@classmethod
237-
def post_make_record(cls, row):
245+
def post_make_record(cls, row, **kwargs):
238246
""" Base does nothing, but a derived class now has a chance to clean up the record prior to db commit """
239247
return row
240248

gtfsdb/model/block.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,13 @@ class Block(Base):
2727

2828
id = Column(Integer, Sequence(None, optional=True), primary_key=True)
2929
sequence = Column(Integer)
30-
block_id = Column(String(255), index=True, nullable=False)
31-
service_id = Column(String(255), index=True, nullable=False)
32-
trip_id = Column(String(255), index=True, nullable=False)
33-
prev_trip_id = Column(String(255))
34-
next_trip_id = Column(String(255))
35-
start_stop_id = Column(String(255), index=True, nullable=False)
36-
end_stop_id = Column(String(255), index=True, nullable=False)
30+
block_id = Column(String(512), index=True, nullable=False)
31+
service_id = Column(String(512), index=True, nullable=False)
32+
trip_id = Column(String(512), index=True, nullable=False)
33+
prev_trip_id = Column(String(512))
34+
next_trip_id = Column(String(512))
35+
start_stop_id = Column(String(512), index=True, nullable=False)
36+
end_stop_id = Column(String(512), index=True, nullable=False)
3737

3838
universal_calendar = relationship(
3939
'UniversalCalendar',

gtfsdb/model/calendar.py

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ class Calendar(Base):
2424
__tablename__ = 'calendar'
2525
__table_args__ = (Index('calendar_ix1', 'start_date', 'end_date'),)
2626

27-
service_id = Column(String(255), primary_key=True, index=True, nullable=False)
27+
service_id = Column(String(512), primary_key=True, index=True, nullable=False)
2828
monday = Column(SmallInteger, nullable=False)
2929
tuesday = Column(SmallInteger, nullable=False)
3030
wednesday = Column(SmallInteger, nullable=False)
@@ -34,7 +34,7 @@ class Calendar(Base):
3434
sunday = Column(SmallInteger, nullable=False)
3535
start_date = Column(Date, nullable=False)
3636
end_date = Column(Date, nullable=False)
37-
service_name = Column(String(255)) # Trillium extension, a human-readable name for the calendar.
37+
service_name = Column(String(512)) # Trillium extension, a human-readable name for the calendar.
3838

3939
def weekday_list(self):
4040
weekday_dict = dict(monday=0, tuesday=1, wednesday=2, thursday=3, friday=4, saturday=5, sunday=6)
@@ -62,7 +62,7 @@ class CalendarDate(Base):
6262

6363
__tablename__ = 'calendar_dates'
6464

65-
service_id = Column(String(255), primary_key=True, index=True, nullable=False)
65+
service_id = Column(String(512), primary_key=True, index=True, nullable=False)
6666
date = Column(Date, primary_key=True, index=True, nullable=False)
6767
exception_type = Column(Integer, nullable=False)
6868

@@ -79,7 +79,7 @@ class UniversalCalendar(Base):
7979
datasource = config.DATASOURCE_DERIVED
8080
__tablename__ = 'universal_calendar'
8181

82-
service_id = Column(String(255), primary_key=True, index=True, nullable=False)
82+
service_id = Column(String(512), primary_key=True, index=True, nullable=False)
8383
date = Column(Date, primary_key=True, index=True, nullable=False)
8484

8585
trips = relationship(
@@ -97,12 +97,26 @@ def load(cls, db, **kwargs):
9797
session.commit()
9898
q = session.query(CalendarDate)
9999
for calendar_date in q:
100-
cd_kwargs = dict(date=calendar_date.date,
101-
service_id=calendar_date.service_id)
100+
cd_kwargs = dict(date=calendar_date.date, service_id=calendar_date.service_id)
102101
if calendar_date.is_addition:
103102
session.merge(cls(**cd_kwargs))
104103
if calendar_date.is_removal:
105-
session.query(cls).filter_by(**cd_kwargs).delete()
104+
try:
105+
# import pdb; pdb.set_trace()
106+
"""
107+
TODO 11-1-2023 - exceptions thrown by executing this delete indicate that the ORM might be in a very funky state ... shows that former agency data is in ORM
108+
109+
2023-10-30 12:21:23,341 - gtfsdb.model.calendar (line 109) - WARNING - (psycopg2.errors.UndefinedTable) invalid reference to FROM-clause entry for table "universal_calendar"
110+
LINE 1: DELETE FROM cat.universal_calendar WHERE canby.universal_cal...
111+
HINT: There is an entry for table "universal_calendar", but it cannot be referenced from this part of the query.
112+
[SQL: DELETE FROM cat.universal_calendar WHERE canby.universal_calendar.date = %(date_1)s AND canby.universal_calendar.service_id = %(service_id_1)s]
113+
[parameters: {'date_1': datetime.date(2024, 1, 1), 'service_id_1': 'c_69855_b_81444_d_31'}]
114+
"""
115+
session.query(cls).filter_by(**cd_kwargs).delete()
116+
except Exception as e:
117+
log.warning(e)
118+
106119
session.commit()
120+
session.flush()
107121
process_time = time.time() - start_time
108122
log.debug('{0}.load ({1:.0f} seconds)'.format(cls.__name__, process_time))

gtfsdb/model/db.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,13 @@ def __init__(self, **kwargs):
2424
url = kwargs.get('database_url', config.DEFAULT_DATABASE_URL)
2525
self.url = url
2626
self.schema = kwargs.get('schema', config.DEFAULT_SCHEMA)
27+
self.feed_id = kwargs.get('feed_id', None)
2728
self.is_geospatial = kwargs.get('is_geospatial', config.DEFAULT_IS_GEOSPATIAL)
2829

2930
"""Order list of class names, used for creating & populating tables"""
3031
from gtfsdb import SORTED_CLASS_NAMES, CURRENT_CLASS_NAMES
3132
self.sorted_class_names = SORTED_CLASS_NAMES
32-
if kwargs.get('current_tables'):
33+
if kwargs.get('current_tables') or kwargs.get('current_tables_all'):
3334
self.sorted_class_names.extend(CURRENT_CLASS_NAMES)
3435
# import pdb; pdb.set_trace()
3536

0 commit comments

Comments
 (0)