Skip to content

Commit

Permalink
bigquery: account for datetime/int types properly
Browse files Browse the repository at this point in the history
BigQuery returns a `datetime`, which we then compare to the
`int` returned from the Phab DB. Convert the `datetime` to
an `int` timestamp before returning.
  • Loading branch information
cgsheeh committed Nov 4, 2024
1 parent 0bb63ae commit a1dbdb3
Showing 1 changed file with 19 additions and 9 deletions.
28 changes: 19 additions & 9 deletions stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import sys
import time
from dataclasses import dataclass
from datetime import datetime
from datetime import datetime, timezone
from enum import IntEnum
from typing import Optional, Any

Expand Down Expand Up @@ -413,7 +413,7 @@ def get_comments(
return comments


def get_last_run_timestamp(bq_client: bigquery.Client) -> Optional[int]:
def get_last_run_timestamp(bq_client: bigquery.Client) -> Optional[datetime]:
"""Get the timestamp of the most recently added entry in BigQuery.
See https://github.com/googleapis/python-bigquery/blob/main/samples/query_script.py
Expand Down Expand Up @@ -470,17 +470,27 @@ def get_time_queries(now: datetime, bq_client: bigquery.Client) -> list:
Dont take the revisions created before the last run
"""
queries = [
DiffDb.Revision.dateCreated < now.timestamp(),
DiffDb.Revision.dateModified < now.timestamp(),
or_(
DiffDb.Revision.dateCreated < now.timestamp(),
DiffDb.Revision.dateModified < now.timestamp(),
),
]
last_run_timestamp = get_last_run_timestamp(bq_client)
last_run_datetime = get_last_run_timestamp(bq_client)

if last_run_datetime:
last_run_datetime = last_run_datetime.replace(tzinfo=timezone.utc)
last_run_timestamp = last_run_datetime.timestamp()

logging.info(
f"Using {last_run_datetime} as the last run date ({last_run_timestamp})."
)

if last_run_timestamp:
logging.info(f"Using {last_run_timestamp} as the last run timestamp.")
queries.extend(
[
DiffDb.Revision.dateCreated > last_run_timestamp,
DiffDb.Revision.dateModified > last_run_timestamp,
or_(
DiffDb.Revision.dateCreated > last_run_timestamp,
DiffDb.Revision.dateModified > last_run_timestamp,
),
]
)
else:
Expand Down

0 comments on commit a1dbdb3

Please sign in to comment.