dlthub-education/workshops/orchestrator_course/example_pipeline.py at 04b853f2fe85136d34a78a3da7b452775a0bcb7d · dlt-hub/dlthub-education · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

import dlt
from dlt.sources.rest_api import RESTAPIConfig, rest_api_source

config: RESTAPIConfig = {
    "client": {
        "base_url": "https://api.github.com",
        "auth": {
            "token": dlt.secrets["sources.access_token"],
        },
        "headers": {
            "Accept": "application/vnd.github+json",
            "X-GitHub-Api-Version": "2022-11-28"
        },
        "paginator": "header_link"
    },
    "resources": [
        {
            "name": "repos",
            "endpoint": {
                "path": "orgs/dlt-hub/repos"
            },
        },
        {
          "name": "contributors",
          "endpoint": {
            "path": "repos/dlt-hub/dlt/contributors",
          },
        },
        {
          "name": "issues",
          "endpoint": {
            "path": "repos/dlt-hub/dlt/issues",
            "params": {
                    "state": "open",  # Only get open issues
                    "sort": "updated",
                    "direction": "desc",
                    "since": "{incremental.start_value}"  # For incremental loading
                },
                "incremental": {
                    "cursor_path": "updated_at",
                    "initial_value": "2025-03-01T00:00:00Z",
                }
          },
        },
        {
          "name": "forks",
          "endpoint": {
            "path": "repos/dlt-hub/dlt/forks",
            "params": {
                    "sort": "oldest",      # Ensures ascending creation order
                    "per_page": 100
                },
                "incremental": {           #backfill
                    "cursor_path": "created_at",
                    "initial_value": "2025-07-01T00:00:00Z",
                    "end_value": "2025-08-01T00:00:00Z",
                    "row_order": "asc"
                }
          },
        },
        {
          "name": "releases",
          "endpoint": {
            "path": "repos/dlt-hub/dlt/releases",
          },
        },
    ],
}

github_source = rest_api_source(config)