File tree 1 file changed +20
-6
lines changed
1 file changed +20
-6
lines changed Original file line number Diff line number Diff line change 18
18
19
19
20
20
def get_my_source () -> ab .Source :
21
- return ab .get_source (
22
- "source-faker" ,
21
+ github_pat = ab .get_secret ("GITHUB_PERSONAL_ACCESS_TOKEN" )
22
+ assert str (github_pat )
23
+ source = ab .get_source (
24
+ "source-github" ,
23
25
config = {
24
- "count" : SCALE ,
25
- "seed" : 1234 ,
26
- "parallelism" : 16 ,
26
+ "repositories" : ["airbytehq/PyAirbyte" ],
27
+ "credentials" : {
28
+ "personal_access_token" : github_pat ,
29
+ },
27
30
},
28
- streams = ["purchases" ],
31
+ # streams=["issues"],
32
+ )
33
+ source .check ()
34
+ source .select_streams (["issues" ])
35
+ return source
36
+
37
+
38
+ def get_cache () -> ab .DuckDBCache :
39
+ return ab .new_local_cache (
40
+ cache_name = "state_cache" ,
29
41
)
30
42
31
43
@@ -48,9 +60,11 @@ def main() -> None:
48
60
source .check ()
49
61
destination = get_my_destination ()
50
62
destination .check ()
63
+ state_cache = get_cache ()
51
64
write_result : ab .WriteResult = destination .write (
52
65
source ,
53
66
cache = False ,
67
+ state_cache = state_cache ,
54
68
)
55
69
print (
56
70
f"Completed writing { write_result .processed_records :,} records "
You can’t perform that action at this time.
0 commit comments