airbyte/.github/workflows/generate-connector-registries.yml at f55dcfe63cdaf3ddf13e0529fac2dead9747a5cf · airbytehq/airbyte · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
name: Generate Connector Registries

on:
  workflow_dispatch:
    inputs:
      gitref:
        description: "The git ref to check out from the repository"
        required: false
        type: string
  schedule:
    - cron: "*/5 * * * *"

permissions:
  contents: read

jobs:
  generate-cloud-registry:
    name: Generate Cloud Registry
    runs-on: ubuntu-24.04

    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@8edcb1bdb4e267140fa742c62e395cd74f332709 # v4.2.2
        with:
          ref: ${{ github.event.inputs.gitref || github.ref }}

      - name: Set up Python
        uses: actions/setup-python@9322b3ca74000aeb2c01eb777b646334015ddd72 # v5.6.0
        with:
          python-version: "3.11"
          check-latest: true
          update-environment: true

      - name: Install Poetry
        uses: snok/install-poetry@d526ede1e27960b7b181a5ac53044f552afdaa38 # v1.4.1
        with:
          version: latest
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install metadata_service
        working-directory: airbyte-ci/connectors/metadata_service/lib
        run: poetry install

      - name: Generate Cloud Registry
        env:
          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          GCS_DEV_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}
          SLACK_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        #   SENTRY_DSN: ${{ secrets.SENTRY_DSN_METADATA_SERVICE }}
        #   SENTRY_TRACES_SAMPLE_RATE: 1.0
        #   SENTRY_ENVIRONMENT: production
        working-directory: airbyte-ci/connectors/metadata_service/lib
        shell: bash
        run: |
          poetry run metadata_service generate-connector-registry prod-airbyte-cloud-connector-metadata-service cloud

  generate-oss-registry:
    name: Generate OSS Registry
    runs-on: ubuntu-24.04

    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@8edcb1bdb4e267140fa742c62e395cd74f332709 # v4.2.2
        with:
          ref: ${{ github.event.inputs.gitref || github.ref }}

      - name: Set up Python
        uses: actions/setup-python@9322b3ca74000aeb2c01eb777b646334015ddd72 # v5.6.0
        with:
          python-version: "3.11"
          check-latest: true
          update-environment: true

      - name: Install Poetry
        uses: snok/install-poetry@d526ede1e27960b7b181a5ac53044f552afdaa38 # v1.4.1
        with:
          version: latest
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install metadata_service
        working-directory: airbyte-ci/connectors/metadata_service/lib
        run: poetry install

      - name: Generate OSS Registry
        env:
          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          GCS_DEV_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}
          SLACK_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        #   SENTRY_DSN: ${{ secrets.SENTRY_DSN_METADATA_SERVICE }}
        #   SENTRY_TRACES_SAMPLE_RATE: 1.0
        #   SENTRY_ENVIRONMENT: production
        working-directory: airbyte-ci/connectors/metadata_service/lib
        shell: bash
        run: |
          poetry run metadata_service generate-connector-registry prod-airbyte-cloud-connector-metadata-service oss

  generate-specs-secrets-mask:
    name: Generate Specs Secrets Mask
    runs-on: ubuntu-24.04
    needs: [generate-cloud-registry, generate-oss-registry]

    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@8edcb1bdb4e267140fa742c62e395cd74f332709 # v4.2.2
        with:
          ref: ${{ github.event.inputs.gitref || github.ref }}

      - name: Set up Python
        uses: actions/setup-python@9322b3ca74000aeb2c01eb777b646334015ddd72 # v5.6.0
        with:
          python-version: "3.11"
          check-latest: true
          update-environment: true

      - name: Install Poetry
        uses: snok/install-poetry@d526ede1e27960b7b181a5ac53044f552afdaa38 # v1.4.1
        with:
          version: latest
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Install metadata_service
        working-directory: airbyte-ci/connectors/metadata_service/lib
        run: poetry install

      - name: Generate Specs Secrets Mask
        env:
          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          GCS_DEV_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}
          SLACK_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        #   SENTRY_DSN: ${{ secrets.SENTRY_DSN_METADATA_SERVICE }}
        #   SENTRY_TRACES_SAMPLE_RATE: 1.0
        #   SENTRY_ENVIRONMENT: production
        working-directory: airbyte-ci/connectors/metadata_service/lib
        shell: bash
        run: |
          poetry run metadata_service generate-specs-secrets-mask prod-airbyte-cloud-connector-metadata-service