From 41e1fbcb6bad222c17820cd87bf3d52488ba32f7 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 15 Apr 2025 23:02:41 -0700 Subject: [PATCH] chore: update source configuration to use GitHub and select 'issues' stream --- examples/run_sync_to_destination_wo_cache.py | 26 +++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/examples/run_sync_to_destination_wo_cache.py b/examples/run_sync_to_destination_wo_cache.py index 1ab07116..f7054f52 100644 --- a/examples/run_sync_to_destination_wo_cache.py +++ b/examples/run_sync_to_destination_wo_cache.py @@ -18,14 +18,26 @@ def get_my_source() -> ab.Source: - return ab.get_source( - "source-faker", + github_pat = ab.get_secret("GITHUB_PERSONAL_ACCESS_TOKEN") + assert str(github_pat) + source = ab.get_source( + "source-github", config={ - "count": SCALE, - "seed": 1234, - "parallelism": 16, + "repositories": ["airbytehq/PyAirbyte"], + "credentials": { + "personal_access_token": github_pat, + }, }, - streams=["purchases"], + # streams=["issues"], + ) + source.check() + source.select_streams(["issues"]) + return source + + +def get_cache() -> ab.DuckDBCache: + return ab.new_local_cache( + cache_name="state_cache", ) @@ -48,9 +60,11 @@ def main() -> None: source.check() destination = get_my_destination() destination.check() + state_cache = get_cache() write_result: ab.WriteResult = destination.write( source, cache=False, + state_cache=state_cache, ) print( f"Completed writing {write_result.processed_records:,} records "