Skip to content

Commit 2914d08

Browse files
committed
update cron schedule to run 4x/day
1 parent 6ecb579 commit 2914d08

File tree

3 files changed

+24
-24
lines changed

3 files changed

+24
-24
lines changed

.github/workflows/update-events-data.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: Scrape Instagram, Update Events DB, Send Newsletter, Update Static Data
22

33
on:
44
schedule:
5-
- cron: '0 11 * * *' # Once daily at noon UTC (8am EST)
5+
- cron: '0 3,12,17,22 * * *' # 11pm, 8am, 1pm, 6pm EST
66
workflow_dispatch: # Optional manual trigger
77
inputs:
88
run_scraper:
@@ -16,7 +16,7 @@ on:
1616
MAX_POSTS:
1717
required: false
1818
type: number
19-
default: 100
19+
default: 20
2020
CUTOFF_DAYS:
2121
required: false
2222
type: number
@@ -28,7 +28,7 @@ jobs:
2828
permissions:
2929
contents: write
3030
env:
31-
MAX_POSTS: ${{ github.event.inputs.MAX_POSTS || '100' }}
31+
MAX_POSTS: ${{ github.event.inputs.MAX_POSTS || '20' }}
3232
CUTOFF_DAYS: ${{ github.event.inputs.CUTOFF_DAYS || '2' }}
3333
PRODUCTION: '1'
3434
DJANGO_SETTINGS_MODULE: 'config.settings.development'

backend/scraping/instagram_feed.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import traceback
1717
from datetime import datetime, timedelta
1818
from pathlib import Path
19+
from dateutil import parser as dateutil_parser
1920

2021
import requests
2122
from requests.exceptions import ReadTimeout, ConnectionError
@@ -203,8 +204,9 @@ def append_event_to_csv(
203204
def insert_event_to_db(event_data, ig_handle, source_url):
204205
"""Map scraped event data to Event model fields, insert to DB"""
205206
title = event_data.get("title")
206-
dtstart = event_data.get("dtstart")
207-
dtend = event_data.get("dtend") or None
207+
dtstart = dateutil_parser.parse(event_data.get("dtstart")).replace(tzinfo=None)
208+
dtend = dateutil_parser.parse(event_data.get("dtend")).replace(tzinfo=None) \
209+
if event_data.get("dtend") else None
208210
source_image_url = event_data.get("source_image_url") or ""
209211
description = event_data.get("description") or ""
210212
location = event_data.get("location")

backend/scraping/logging_config.py

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,27 +9,25 @@
99

1010
def setup_logging():
1111
root_logger = logging.getLogger()
12-
if not getattr(root_logger, "_wat2do_configured", False):
13-
for handler in list(root_logger.handlers):
14-
root_logger.removeHandler(handler)
12+
if getattr(root_logger, "_wat2do_configured", False):
13+
return
14+
for handler in list(root_logger.handlers):
15+
root_logger.removeHandler(handler)
1516

16-
logging.getLogger("urllib3").setLevel(logging.WARNING)
17-
logging.getLogger("requests").setLevel(logging.WARNING)
18-
logging.getLogger("botocore").setLevel(logging.WARNING)
19-
logging.getLogger("httpcore").setLevel(logging.WARNING)
20-
logging.getLogger("openai").setLevel(logging.WARNING)
21-
logging.getLogger("PIL").setLevel(logging.INFO)
17+
logging.getLogger("urllib3").setLevel(logging.WARNING)
18+
logging.getLogger("requests").setLevel(logging.WARNING)
19+
logging.getLogger("botocore").setLevel(logging.WARNING)
20+
logging.getLogger("httpcore").setLevel(logging.WARNING)
21+
logging.getLogger("openai").setLevel(logging.WARNING)
22+
logging.getLogger("PIL").setLevel(logging.INFO)
2223

23-
fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
24-
handlers = [
25-
logging.StreamHandler(sys.stdout),
26-
logging.FileHandler(LOG_FILE, encoding="utf-8"),
27-
]
28-
logging.basicConfig(level=logging.DEBUG, format=fmt, handlers=handlers)
29-
for h in handlers:
30-
root_logger.addHandler(h)
31-
root_logger.setLevel(logging.DEBUG)
32-
root_logger._wat2do_configured = True
24+
fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
25+
handlers = [
26+
logging.StreamHandler(sys.stdout),
27+
logging.FileHandler(LOG_FILE, encoding="utf-8"),
28+
]
29+
logging.basicConfig(level=logging.DEBUG, format=fmt, handlers=handlers)
30+
root_logger._wat2do_configured = True
3331

3432

3533
setup_logging()

0 commit comments

Comments
 (0)