fix(source-twilio): Resume messages from saved cursor #1568
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Connector Active Progressive Rollout Checks | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| - reopened | |
| - edited | |
| - ready_for_review | |
| workflow_dispatch: | |
| inputs: | |
| repo: | |
| description: "The repository name" | |
| required: false | |
| default: "airbytehq/airbyte" | |
| type: string | |
| pr: | |
| description: "The pull request number to inspect and comment on." | |
| required: true | |
| type: number | |
| permissions: | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| concurrency: | |
| group: rollout-gate-${{ github.event.pull_request.number || inputs.pr || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| generate-matrix: | |
| name: Generate Connector Matrix | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Resolve workflow inputs | |
| id: vars | |
| env: | |
| INPUT_REPO: ${{ inputs.repo }} | |
| INPUT_PR: ${{ inputs.pr }} | |
| EVENT_PR_NUMBER: ${{ github.event.pull_request.number }} | |
| PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_HEAD_REF: ${{ github.head_ref }} | |
| GITHUB_REF_NAME: ${{ github.ref_name }} | |
| run: | | |
| pr_number="${INPUT_PR:-${EVENT_PR_NUMBER}}" | |
| checkout_repository="${INPUT_REPO:-${PR_HEAD_REPO:-${GITHUB_REPOSITORY}}}" | |
| pr_repository="${INPUT_REPO:-${GITHUB_REPOSITORY}}" | |
| if [[ -n "${INPUT_PR}" ]]; then | |
| ref="refs/pull/${INPUT_PR}/head" | |
| else | |
| ref="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" | |
| fi | |
| is_fork="true" | |
| if [[ "${checkout_repository}" == airbytehq/* ]]; then | |
| is_fork="false" | |
| fi | |
| echo "pr-number=${pr_number}" | tee -a "${GITHUB_OUTPUT}" | |
| echo "pr-repository=${pr_repository}" | tee -a "${GITHUB_OUTPUT}" | |
| echo "checkout-repository=${checkout_repository}" | tee -a "${GITHUB_OUTPUT}" | |
| echo "ref=${ref}" | tee -a "${GITHUB_OUTPUT}" | |
| echo "is-fork=${is_fork}" | tee -a "${GITHUB_OUTPUT}" | |
| - name: Install uv | |
| if: steps.vars.outputs.pr-number != '' | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 | |
| with: | |
| ignore-empty-workdir: true | |
| - name: Install Ops CLI | |
| if: steps.vars.outputs.pr-number != '' | |
| run: uv tool install 'airbyte-internal-ops>=0.55.0' | |
| - name: Resolve connector matrix | |
| id: connector-matrix | |
| env: | |
| PR_NUMBER: ${{ steps.vars.outputs.pr-number }} | |
| PR_REPOSITORY: ${{ steps.vars.outputs.pr-repository }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| if [[ -n "${PR_NUMBER}" ]]; then | |
| repo_owner="${PR_REPOSITORY%%/*}" | |
| repo_name="${PR_REPOSITORY#*/}" | |
| connectors_matrix="$( | |
| airbyte-ops gh connector list \ | |
| --modified-only \ | |
| --pr "${PR_NUMBER}" \ | |
| --owner "${repo_owner}" \ | |
| --repo "${repo_name}" \ | |
| --gh-token "${GITHUB_TOKEN}" \ | |
| --output-format json-gh-matrix | |
| )" | |
| else | |
| echo "::error::Auto-detecting modified connectors requires a pull request number." | |
| exit 1 | |
| fi | |
| echo "connectors_matrix=${connectors_matrix}" | tee -a "${GITHUB_OUTPUT}" | |
| outputs: | |
| connectors-matrix: ${{ steps.connector-matrix.outputs.connectors_matrix }} | |
| checkout-repository: ${{ steps.vars.outputs.checkout-repository }} | |
| checkout-ref: ${{ steps.vars.outputs.ref }} | |
| progressive-rollout-gate: | |
| needs: [generate-matrix] | |
| strategy: | |
| matrix: ${{ fromJson(needs.generate-matrix.outputs.connectors-matrix) }} | |
| max-parallel: 10 | |
| fail-fast: false | |
| name: ${{ matrix.connector || 'No-Op' }} Progressive Rollout Gate | |
| runs-on: ubuntu-24.04 | |
| env: | |
| ROLLOUT_ACK_CHECKBOX_TEXT: "(Click to Approve:) Bypass the active progressive rollout warning for ${{ matrix.connector }} in the PR comment" | |
| GCP_PROD_DB_ACCESS_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr }} | |
| steps: | |
| - name: Checkout Airbyte | |
| if: matrix.connector | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| repository: ${{ needs.generate-matrix.outputs.checkout-repository }} | |
| ref: ${{ needs.generate-matrix.outputs.checkout-ref }} | |
| submodules: true # Needed for airbyte-enterprise connectors (no-op otherwise) | |
| fetch-depth: 1 | |
| - name: Install uv | |
| if: matrix.connector | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 | |
| - name: Install Ops CLI | |
| if: matrix.connector | |
| run: uv tool install airbyte-internal-ops | |
| - name: Install Cloud SQL Proxy | |
| if: matrix.connector | |
| env: | |
| CLOUD_SQL_PROXY_VERSION: v2.15.0 | |
| CLOUD_SQL_PROXY_SHA256: cf3e9d069ea0d09cdfddce77daa36811bb0b963b1169e9c3f1586433ca7325fa | |
| run: | | |
| set -euo pipefail | |
| BINARY_NAME="cloud-sql-proxy.linux.amd64" | |
| BASE_URL="https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/${CLOUD_SQL_PROXY_VERSION}" | |
| curl -fsSLO "${BASE_URL}/${BINARY_NAME}" | |
| echo "${CLOUD_SQL_PROXY_SHA256} ${BINARY_NAME}" | sha256sum -c - | |
| chmod +x "${BINARY_NAME}" | |
| sudo mv "${BINARY_NAME}" /usr/local/bin/cloud-sql-proxy | |
| cloud-sql-proxy --version | |
| - name: Start Cloud SQL Proxy | |
| if: matrix.connector | |
| env: | |
| # Local Cloud SQL Proxy port for prod DB queries. | |
| DB_PORT: "15432" | |
| run: airbyte-ops cloud db start-proxy --port "${DB_PORT}" | |
| - name: Get rollout status | |
| if: matrix.connector | |
| id: rollout-status | |
| env: | |
| CONNECTOR_NAME: ${{ matrix.connector }} | |
| # Local Cloud SQL Proxy port for prod DB queries. | |
| DB_PORT: "15432" | |
| run: | | |
| { | |
| echo "json<<EOF" | |
| airbyte-ops registry progressive-rollout status "${CONNECTOR_NAME}" --repo-path . | |
| echo "EOF" | |
| } | tee -a "${GITHUB_OUTPUT}" | |
| - name: Get master connector version | |
| if: matrix.connector | |
| id: master-version | |
| env: | |
| CONNECTOR_NAME: ${{ matrix.connector }} | |
| run: | | |
| VERSION=$(airbyte-ops gh connector get-version --name "${CONNECTOR_NAME}" --ref master) | |
| echo "version=${VERSION}" | tee -a "${GITHUB_OUTPUT}" | |
| - name: Resolve rollout gate state | |
| if: matrix.connector | |
| id: rollout-gate-state | |
| env: | |
| ROLLOUT_STATUS_JSON: ${{ steps.rollout-status.outputs.json }} | |
| MASTER_VERSION: ${{ steps.master-version.outputs.version }} | |
| run: | | |
| retool_base_url="https://airbyte.retool.com/apps/7aa774a0-8750-11ef-896b-236cbc10fa7e/Connector%20Rollout%20Manager" | |
| actor_definition_id="$(jq -r '.actor_definition_id // empty' <<< "${ROLLOUT_STATUS_JSON}")" | |
| retool_url="${retool_base_url}" | |
| if [[ -n "${actor_definition_id}" ]]; then | |
| retool_url="${retool_base_url}?actorDefinitionId=${actor_definition_id}" | |
| fi | |
| active_rollout='[.rollouts[]? | select(.state | IN("initialized", "workflow_started", "in_progress", "paused", "finalizing", "errored"))][0] // {}' | |
| updated_by_name="$(jq -r "(${active_rollout}).updated_by_user_name // empty" <<< "${ROLLOUT_STATUS_JSON}")" | |
| updated_by_email="$(jq -r "(${active_rollout}).updated_by_user_email // empty" <<< "${ROLLOUT_STATUS_JSON}")" | |
| updated_by_display="unknown" | |
| if [[ -n "${updated_by_name}" && -n "${updated_by_email}" ]]; then | |
| updated_by_display="${updated_by_name} <${updated_by_email}>" | |
| elif [[ -n "${updated_by_email}" ]]; then | |
| updated_by_display="${updated_by_email}" | |
| elif [[ -n "${updated_by_name}" ]]; then | |
| updated_by_display="${updated_by_name}" | |
| fi | |
| has_active_rollout="$(jq -r '.has_active_rollout' <<< "${ROLLOUT_STATUS_JSON}")" | |
| has_master_rc_marker="false" | |
| if [[ "${MASTER_VERSION}" == *"-rc."* ]]; then | |
| has_master_rc_marker="true" | |
| fi | |
| requires_ack="false" | |
| if [[ "${has_active_rollout}" == "true" || "${has_master_rc_marker}" == "true" ]]; then | |
| requires_ack="true" | |
| fi | |
| echo "has-active-rollout=${has_active_rollout}" | tee -a "${GITHUB_OUTPUT}" | |
| echo "has-master-rc-marker=${has_master_rc_marker}" | tee -a "${GITHUB_OUTPUT}" | |
| echo "requires-ack=${requires_ack}" | tee -a "${GITHUB_OUTPUT}" | |
| echo "rollout-updated-by=${updated_by_display}" | tee -a "${GITHUB_OUTPUT}" | |
| echo "retool-url=${retool_url}" | tee -a "${GITHUB_OUTPUT}" | |
| - name: Detect rollout bypass checkbox | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' && | |
| github.event_name == 'pull_request' | |
| id: rollout-bypass | |
| uses: AhmedBaset/checklist@ecbf0bacb0d5d1fe41131db29a44984fec266017 # v3 | |
| with: | |
| token: ${{ github.token }} | |
| - name: Detect rollout bypass checkbox for dispatched PR | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' && | |
| github.event_name == 'workflow_dispatch' && | |
| env.PR_NUMBER != '' | |
| id: rollout-bypass-dispatch | |
| uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 | |
| env: | |
| ROLLOUT_ACK_CHECKBOX_TEXT: ${{ env.ROLLOUT_ACK_CHECKBOX_TEXT }} | |
| PR_NUMBER: ${{ env.PR_NUMBER }} | |
| with: | |
| script: | | |
| const pullNumber = Number(process.env.PR_NUMBER); | |
| const checkboxText = process.env.ROLLOUT_ACK_CHECKBOX_TEXT; | |
| const { owner, repo } = context.repo; | |
| const { data: pull } = await github.rest.pulls.get({ | |
| owner, | |
| repo, | |
| pull_number: pullNumber, | |
| }); | |
| const lines = (pull.body || "").split(/\r?\n/).map((line) => line.trim()); | |
| const checked = lines.some((line) => /^- \[[xX]\]/.test(line) && line.includes(checkboxText)); | |
| const unchecked = lines.some((line) => line.startsWith("- [ ]") && line.includes(checkboxText)); | |
| core.setOutput("approved", checked.toString()); | |
| core.setOutput("exists", (checked || unchecked).toString()); | |
| - name: Resolve rollout bypass checkbox state | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' | |
| id: bypass-state | |
| env: | |
| APPROVED: ${{ contains(steps.rollout-bypass.outputs.checked, env.ROLLOUT_ACK_CHECKBOX_TEXT) || steps.rollout-bypass-dispatch.outputs.approved == 'true' }} | |
| EXISTS: ${{ contains(steps.rollout-bypass.outputs.checked, env.ROLLOUT_ACK_CHECKBOX_TEXT) || contains(steps.rollout-bypass.outputs.unchecked, env.ROLLOUT_ACK_CHECKBOX_TEXT) || steps.rollout-bypass-dispatch.outputs.exists == 'true' }} | |
| run: | | |
| echo "approved=${APPROVED}" | tee -a $GITHUB_OUTPUT | |
| echo "exists=${EXISTS}" | tee -a $GITHUB_OUTPUT | |
| - name: Find progressive rollout gate comment | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' && | |
| env.PR_NUMBER != '' | |
| id: find-rollout-comment | |
| uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 | |
| with: | |
| issue-number: ${{ env.PR_NUMBER }} | |
| comment-author: github-actions[bot] | |
| body-includes: "<!-- progressive-rollout-gate:${{ matrix.connector }} -->" | |
| - name: Create progressive rollout gate placeholder comment | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' && | |
| env.PR_NUMBER != '' && | |
| steps.find-rollout-comment.outputs.comment-id == '' | |
| id: create-rollout-comment | |
| uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 | |
| with: | |
| issue-number: ${{ env.PR_NUMBER }} | |
| edit-mode: replace | |
| body: | | |
| <!-- progressive-rollout-gate:${{ matrix.connector }} --> | |
| > **Progressive rollout gate for `${{ matrix.connector }}`** | |
| > | |
| > Checking for active rollouts. | |
| > | |
| > [Workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| - name: Resolve rollout comment link | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' | |
| id: rollout-comment | |
| env: | |
| EXISTING_COMMENT_ID: ${{ steps.find-rollout-comment.outputs.comment-id }} | |
| CREATED_COMMENT_ID: ${{ steps.create-rollout-comment.outputs.comment-id }} | |
| REPOSITORY: ${{ github.repository }} | |
| run: | | |
| COMMENT_ID="${EXISTING_COMMENT_ID:-${CREATED_COMMENT_ID}}" | |
| echo "id=${COMMENT_ID}" | tee -a $GITHUB_OUTPUT | |
| echo "url=https://github.com/${REPOSITORY}/pull/${PR_NUMBER}#issuecomment-${COMMENT_ID}" | tee -a $GITHUB_OUTPUT | |
| - name: Add rollout bypass checkbox to PR description | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' && | |
| github.event_name == 'pull_request' && | |
| steps.bypass-state.outputs.exists != 'true' | |
| uses: bcgov/action-pr-description-add@14338bfe0278ead273b3c1189e5aa286ff6709c4 # v2.0.0 | |
| with: | |
| github_token: ${{ github.token }} | |
| add_markdown: | | |
| > [!IMPORTANT] | |
| > Active progressive rollout warning for ${{ matrix.connector }}. | |
| - [ ] (Click to Approve:) Bypass the active progressive rollout warning for ${{ matrix.connector }} in the PR comment [here](${{ steps.rollout-comment.outputs.url }}). | |
| - name: Add rollout bypass checkbox to dispatched PR description | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' && | |
| github.event_name == 'workflow_dispatch' && | |
| env.PR_NUMBER != '' && | |
| steps.bypass-state.outputs.exists != 'true' | |
| uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 | |
| env: | |
| PR_NUMBER: ${{ env.PR_NUMBER }} | |
| ROLLOUT_ACK_CHECKBOX_TEXT: ${{ env.ROLLOUT_ACK_CHECKBOX_TEXT }} | |
| ROLLOUT_COMMENT_URL: ${{ steps.rollout-comment.outputs.url }} | |
| with: | |
| script: | | |
| const pullNumber = Number(process.env.PR_NUMBER); | |
| const checkboxText = process.env.ROLLOUT_ACK_CHECKBOX_TEXT; | |
| const markdown = [ | |
| "> [!IMPORTANT]", | |
| "> Active progressive rollout warning for ${{ matrix.connector }}.", | |
| "", | |
| `- [ ] ${checkboxText} [here](${process.env.ROLLOUT_COMMENT_URL}).`, | |
| ].join("\n"); | |
| const { owner, repo } = context.repo; | |
| const { data: pull } = await github.rest.pulls.get({ | |
| owner, | |
| repo, | |
| pull_number: pullNumber, | |
| }); | |
| const body = pull.body || ""; | |
| if (body.includes(checkboxText)) { | |
| core.info("Rollout bypass checkbox already exists in PR description."); | |
| return; | |
| } | |
| const nextBody = body.trimEnd() ? `${body.trimEnd()}\n\n${markdown}` : markdown; | |
| await github.rest.pulls.update({ | |
| owner, | |
| repo, | |
| pull_number: pullNumber, | |
| body: nextBody, | |
| }); | |
| - name: Render progressive rollout gate comment | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' && | |
| env.PR_NUMBER != '' && | |
| steps.rollout-comment.outputs.id != '' | |
| id: rollout-comment-template | |
| uses: chuhlomin/render-template@f828bb5c72a3e3af89cb79808cea490166c6f1ce # v1.4 | |
| with: | |
| template: .github/progressive-rollout-gate-comment.md | |
| vars: | | |
| connector: ${{ matrix.connector }} | |
| active_rollout: ${{ fromJSON(steps.rollout-status.outputs.json).has_active_rollout }} | |
| rollout_docker_image_tag: ${{ fromJSON(steps.rollout-status.outputs.json).rollouts[0].rollout_docker_image_tag || 'none' }} | |
| rollout_state: ${{ fromJSON(steps.rollout-status.outputs.json).rollouts[0].state || 'none' }} | |
| master_version: ${{ steps.master-version.outputs.version }} | |
| master_rc_marker: ${{ contains(steps.master-version.outputs.version, '-rc.') }} | |
| rollout_updated_by: ${{ steps.rollout-gate-state.outputs.rollout-updated-by }} | |
| retool_url: ${{ steps.rollout-gate-state.outputs.retool-url }} | |
| bypass_ack_checked: ${{ steps.bypass-state.outputs.approved }} | |
| ack_checkbox_text: ${{ env.ROLLOUT_ACK_CHECKBOX_TEXT }} | |
| rollout_comment_url: ${{ steps.rollout-comment.outputs.url }} | |
| workflow_run_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - name: Update progressive rollout gate comment | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.requires-ack == 'true' && | |
| steps.rollout-comment.outputs.id != '' | |
| uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 | |
| with: | |
| comment-id: ${{ steps.rollout-comment.outputs.id }} | |
| issue-number: ${{ env.PR_NUMBER }} | |
| edit-mode: replace | |
| body: ${{ steps.rollout-comment-template.outputs.result }} | |
| - name: Fail if active rollout is not acknowledged | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.has-active-rollout == 'true' && | |
| steps.bypass-state.outputs.approved != 'true' | |
| run: | | |
| echo "::error::${{ matrix.connector }} has an active progressive rollout. Check the bypass box in the PR description to acknowledge and rerun this workflow." | |
| exit 1 | |
| - name: Fail if master has RC marker and is not acknowledged | |
| if: > | |
| matrix.connector && | |
| steps.rollout-gate-state.outputs.has-master-rc-marker == 'true' && | |
| steps.bypass-state.outputs.approved != 'true' | |
| run: | | |
| echo "::error::${{ matrix.connector }} has an RC marker on master (${{ steps.master-version.outputs.version }}). Check the bypass box in the PR description to acknowledge and rerun this workflow." | |
| exit 1 | |
| progressive-rollout-gate-summary: | |
| name: Connector Active Progressive Rollout Checks Summary | |
| if: ${{ !cancelled() }} | |
| needs: | |
| - generate-matrix | |
| - progressive-rollout-gate | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Evaluate Status | |
| if: > | |
| needs.generate-matrix.result == 'success' && | |
| (needs.progressive-rollout-gate.result == 'success' || | |
| needs.progressive-rollout-gate.result == 'skipped') | |
| run: echo "Progressive rollout checks passed." | |
| - name: Evaluate Failure Status | |
| if: > | |
| needs.generate-matrix.result != 'success' || | |
| (needs.progressive-rollout-gate.result != 'success' && | |
| needs.progressive-rollout-gate.result != 'skipped') | |
| run: | | |
| echo "::error::Progressive rollout checks failed." | |
| exit 1 |