Skip to content
ci

Conflict fix #105

Sign in to view logs

Conflict fix

Conflict fix #105

Workflow file for this run

.github/workflows/ci.yml at 9efda56
name: ci
on:
push:
branches:
- master
pull_request:
repository_dispatch:
types: [test-with-secrets-command]
defaults:
run:
shell: bash --noprofile --norc -euo pipefail {0}
env:
# An envar that signals to tests we are executing in the CI environment
CONTINUOUS_INTEGRATION: true
# allow overriding Maven command
MAVEN: ./mvnw
MAVEN_OPTS: "-Xmx512M -XX:+ExitOnOutOfMemoryError"
MAVEN_INSTALL_OPTS: "-Xmx3G -XX:+ExitOnOutOfMemoryError"
MAVEN_FAST_INSTALL: "-B -V -T 1C -DskipTests -Dmaven.source.skip=true -Dair.check.skip-all"
MAVEN_COMPILE_COMMITS: "-B --quiet -T 1C -DskipTests -Dmaven.source.skip=true -Dair.check.skip-all=true -Dmaven.javadoc.skip=true --no-snapshot-updates --no-transfer-progress"
MAVEN_GIB: "-P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.event.repository.default_branch }}"
MAVEN_TEST: "-B -Dmaven.source.skip=true -Dair.check.skip-all --fail-at-end -P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.event.repository.default_branch }}"
# Testcontainers kills image pulls if they don't make progress for > 30s and retries for 2m before failing. This means
# that if an image doesn't download all it's layers within ~2m then any other concurrent pull will be killed because
# the Docker daemon only downloads 3 layers concurrently which prevents the other pull from making any progress.
# This value should be greater than the time taken for the longest image pull.
TESTCONTAINERS_PULL_PAUSE_TIMEOUT: 600
# used by actions/cache to retry the download after this time: https://github.com/actions/cache/blob/main/workarounds.md#cache-segment-restore-timeout
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
CI_SKIP_SECRETS_PRESENCE_CHECKS: ${{ secrets.CI_SKIP_SECRETS_PRESENCE_CHECKS }}
SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }}
PTL_TMP_DOWNLOAD_PATH: /tmp/pt_java_downloads
# Cancel previous PR builds.
concurrency:
# Cancel all workflow runs except latest within a concurrency group. This is achieved by defining a concurrency group for the PR.
# Non-PR builds have singleton concurrency groups.
# When triggered by the repository_dispatch, add the expected SHA to avoid cancelling the run from the PR.
group: |
workflow=${{ github.workflow }},
pr_number=${{ github.event_name == 'pull_request' && github.event.number || 'NA' }},
dispatch_sha=${{ github.event_name == 'repository_dispatch' && github.event.client_payload.slash_command.args.named.sha || 'NA' }},
commit_sha=${{ github.event_name != 'pull_request' && github.event_name != 'repository_dispatch' && github.sha || 'NA' }}
cancel-in-progress: true
jobs:
path-filters:
runs-on: ubuntu-latest
outputs:
docs: ${{ steps.filter.outputs.docs }}
non_docs: ${{ steps.filter.outputs.non_docs }}
steps:
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
id: filter
if: github.event_name == 'pull_request'
with:
# Note: `docs` output is currently unused. The docs changes are tested by maven-checks job, leveraging GIB impact analysis.
# TODO remove use of non_docs filters and remove `path-filters` job. Use GIB impact analysis to guard job skipping.
filters: |
docs: 'docs/**'
non_docs: '!docs/**'
maven-checks:
runs-on: ubuntu-latest
name: maven-checks ${{ matrix.java-version }}
strategy:
fail-fast: false
matrix:
include:
- { java-version: 25, cache: 'true', cleanup-node: true }
- { java-version: 26-ea, cache: 'restore', cleanup-node: true }
timeout-minutes: 45
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
timeout-minutes: 15
with:
cache: ${{ matrix.cache }}
java-version: ${{ matrix.java-version }}
cleanup-node: true
- name: Check SPI backward compatibility
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} -pl :trino-spi -am
$MAVEN clean verify -B --strict-checksums -DskipTests -pl :trino-spi
- name: Maven Checks
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean verify -B --strict-checksums -V -T 1C -DskipTests -P ci
- name: Remove Trino from local Maven repo to avoid caching it
# Avoid caching artifacts built in this job, cache should only include dependencies
if: steps.cache.outputs.cache-hit != 'true' && matrix.cache == 'true'
run: rm -rf ~/.m2/repository/io/trino/trino-*
error-prone-checks:
needs: path-filters
if: github.event_name != 'pull_request' || needs.path-filters.outputs.non_docs == 'true'
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
timeout-minutes: 15
with:
cache: restore
java-version: 25
- name: Maven Install
run: |
# build everything to make sure dependencies of impacted modules are present
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -pl '!:trino-docs,!:trino-server'
- name: Error Prone Checks
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
# Skip checks, these are run in `maven-checks` job and e.g. checkstyle is expensive.
$MAVEN ${MAVEN_TEST} -T 1C clean compile test-compile -DskipTests -Dair.check.skip-all=true ${MAVEN_GIB} -Dgib.buildUpstream=never -P errorprone-compiler \
-pl '!:trino-docs,!:trino-server'
build-test-matrix:
needs: path-filters
if: github.event_name != 'pull_request' || needs.path-filters.outputs.non_docs == 'true'
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
timeout-minutes: 15
with:
cache: restore
- name: Update PR check
uses: ./.github/actions/update-check
if: >-
github.event_name == 'repository_dispatch' &&
github.event.client_payload.slash_command.args.named.sha != '' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha
with:
pull_request_number: ${{ github.event.client_payload.pull_request.number }}
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Maven validate
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -P disable-check-spi-dependencies
- name: Set matrix
id: set-matrix
run: |
# GIB doesn't run on master, so make sure the file always exist
touch gib-impacted.log
cat <<EOF > .github/test-matrix.yaml
include:
- { modules: plugin/trino-teradata }
- { modules: plugin/trino-teradata, profile: clearscape-tests }
EOF
./.github/bin/build-matrix-from-impacted.py -v -i gib-impacted.log -m .github/test-matrix.yaml -o matrix.json
echo "Matrix: $(jq '.' matrix.json)"
echo "matrix=$(jq -c '.' matrix.json)" >> $GITHUB_OUTPUT
test:
runs-on: 'ubuntu-latest'
needs: build-test-matrix
if: needs.build-test-matrix.outputs.matrix != '{}'
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.build-test-matrix.outputs.matrix) }}
timeout-minutes: 120
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
timeout-minutes: 15
with:
cache: restore
cleanup-node: ${{ format('{0}', matrix.modules == 'plugin/trino-singlestore' || matrix.modules == 'plugin/trino-exasol' || matrix.modules == 'plugin/trino-oracle') }}
java-version: ${{ matrix.jdk != '' && matrix.jdk || '25' }}
- name: Maven Install
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -am -pl "${{ matrix.modules }}"
- name: Maven Tests
id: tests
if: >-
! (contains(matrix.modules, 'trino-bigquery') && contains(matrix.profile, 'cloud-tests-2'))
&& ! (contains(matrix.modules, 'trino-delta-lake') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-iceberg') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-redshift') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-redshift') && contains(matrix.profile, 'fte-tests'))
&& ! (contains(matrix.modules, 'trino-snowflake') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-filesystem-azure') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-filesystem-gcs') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-filesystem-s3') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-hdfs') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-teradata') && contains(matrix.profile, 'clearscape-tests'))
run: $MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ matrix.profile != '' && format('-P {0}', matrix.profile) || '' }}
# Additional tests for selected modules
- name: HDFS file system cache isolated JVM tests
id: tests-hdfs-isolated
if: contains(matrix.modules, 'trino-hdfs')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-hdfs -P test-isolated-jvm-suites
- name: Hadoop FileSystem Cloud Tests
id: tests-hdfs
env:
AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
S3_BUCKET_ENDPOINT: "s3.${{ vars.TRINO_AWS_REGION }}.amazonaws.com"
if: >-
contains(matrix.modules, 'trino-hdfs') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-hdfs -P cloud-tests
- name: S3 FileSystem Cloud Tests
id: tests-s3
env:
AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
if: >-
contains(matrix.modules, 'trino-filesystem-s3') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '')
run: |
# Create an empty S3 bucket for S3 filesystem cloud tests and add the bucket name to GitHub environment variables
.github/bin/s3/setup-empty-s3-bucket.sh
EMPTY_S3_BUCKET=$(cat .github/bin/s3/.bucket-identifier)
export EMPTY_S3_BUCKET
$MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ format('-P {0}', matrix.profile) }}
- name: Cleanup ephemeral S3 buckets
env:
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
# Cancelled workflows may not have cleaned up the ephemeral bucket
if: always()
run: .github/bin/s3/delete-s3-bucket.sh || true
- name: Azure FileSystem Cloud Tests
id: tests-azure
env:
ABFS_FLAT_ACCOUNT: ${{ vars.AZURE_ABFS_FLAT_ACCOUNT }}
ABFS_FLAT_ACCESS_KEY: ${{ secrets.AZURE_ABFS_FLAT_ACCESS_KEY }}
ABFS_HIERARCHICAL_ACCOUNT: ${{ vars.AZURE_ABFS_HIERARCHICAL_ACCOUNT }}
ABFS_HIERARCHICAL_ACCESS_KEY: ${{ secrets.AZURE_ABFS_HIERARCHICAL_ACCESS_KEY }}
ABFS_OAUTH_TENANT_ID: ${{ vars.AZURE_ABFS_OAUTH_TENANT_ID }}
ABFS_OAUTH_CLIENT_ID: ${{ vars.AZURE_ABFS_OAUTH_CLIENT_ID }}
ABFS_OAUTH_CLIENT_SECRET: ${{ secrets.AZURE_ABFS_OAUTH_CLIENT_SECRET }}
# Run tests only if any of the secrets are present
if: >-
contains(matrix.modules, 'trino-filesystem-azure') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.ABFS_FLAT_ACCESS_KEY != '' || env.ABFS_HIERARCHICAL_ACCESS_KEY != '' || env.ABFS_OAUTH_CLIENT_SECRET != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ format('-P {0}', matrix.profile) }}
- name: GCS FileSystem Cloud Tests
id: tests-gcs
env:
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
if: >-
contains(matrix.modules, 'trino-filesystem-gcs') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.GCP_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ format('-P {0}', matrix.profile) }}
- name: Cloud Delta Lake Tests
id: tests-delta
# Cloud tests are separate because they are time intensive, requiring cross-cloud network communication
env:
ABFS_CONTAINER: ${{ vars.AZURE_ABFS_HIERARCHICAL_CONTAINER }}
ABFS_ACCOUNT: ${{ vars.AZURE_ABFS_HIERARCHICAL_ACCOUNT }}
ABFS_ACCESSKEY: ${{ secrets.AZURE_ABFS_HIERARCHICAL_ACCESS_KEY }}
AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
# Run tests if any of the secrets is present. Do not skip tests when one secret renamed, or secret name has a typo.
if: >-
contains(matrix.modules, 'trino-delta-lake') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.ABFS_ACCOUNT != '' || env.ABFS_CONTAINER != '' || env.ABFS_ACCESSKEY != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '' || env.GCP_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} ${{ format('-P {0}', matrix.profile) }} -pl :trino-delta-lake \
-Dtesting.azure-abfs-container="${ABFS_CONTAINER}" \
-Dtesting.azure-abfs-account="${ABFS_ACCOUNT}" \
-Dtesting.azure-abfs-access-key="${ABFS_ACCESSKEY}" \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.gcp-credentials-key="${GCP_CREDENTIALS_KEY}"
- name: Cloud BigQuery Tests
id: tests-bq
env:
BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
BIGQUERY_TESTING_BIGLAKE_CONNECTION_ID: ${{ vars.BIGQUERY_TESTING_BIGLAKE_CONNECTION_ID }}
BIGQUERY_TESTING_PROJECT_ID: ${{ vars.BIGQUERY_TESTING_PROJECT_ID }}
BIGQUERY_TESTING_PARENT_PROJECT_ID: ${{ vars.BIGQUERY_TESTING_PARENT_PROJECT_ID }}
if: matrix.modules == 'plugin/trino-bigquery' && !contains(matrix.profile, 'cloud-tests-2') && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.BIGQUERY_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-1 \
-Dtesting.bigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}" \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.bigquery-connection-id="${BIGQUERY_TESTING_BIGLAKE_CONNECTION_ID}" \
-Dtesting.bigquery-project-id="${BIGQUERY_TESTING_PROJECT_ID}" \
-Dtesting.bigquery-parent-project-id="${BIGQUERY_TESTING_PARENT_PROJECT_ID}"
- name: Cloud BigQuery Smoke Tests
id: tests-bq-smoke
env:
BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
if: matrix.modules == 'plugin/trino-bigquery' && contains(matrix.profile, 'cloud-tests-2') && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.BIGQUERY_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-2 \
-Dtesting.bigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}" \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.alternate-bq-project-id=bigquery-cicd-alternate
- name: Cloud BigQuery Case Insensitive Mapping Tests
id: tests-bq-ci
env:
BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY }}
if: matrix.modules == 'plugin/trino-bigquery' && !contains(matrix.profile, 'cloud-tests-2') && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-case-insensitive-mapping -Dtesting.bigquery.credentials-key="${BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY}"
- name: Cloud Snowflake Tests
id: tests-snowflake
env:
SNOWFLAKE_URL: ${{ vars.SNOWFLAKE_URL }}
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
if: matrix.modules == 'plugin/trino-snowflake' && contains(matrix.profile, 'cloud-tests') && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.SNOWFLAKE_URL != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-snowflake -Pcloud-tests \
-Dsnowflake.test.server.url="${SNOWFLAKE_URL}" \
-Dsnowflake.test.server.user="${SNOWFLAKE_USER}" \
-Dsnowflake.test.server.password="${SNOWFLAKE_PASSWORD}" \
-Dsnowflake.test.server.database="${SNOWFLAKE_DATABASE}" \
-Dsnowflake.test.server.role="${SNOWFLAKE_ROLE}" \
-Dsnowflake.test.server.warehouse="${SNOWFLAKE_WAREHOUSE}"
- name: Iceberg Cloud Tests
id: tests-iceberg
env:
AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
S3_TABLES_BUCKET: ${{ vars.TRINO_S3_TABLES_BUCKET }}
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
ABFS_CONTAINER: ${{ vars.AZURE_ABFS_HIERARCHICAL_CONTAINER }}
ABFS_ACCOUNT: ${{ vars.AZURE_ABFS_HIERARCHICAL_ACCOUNT }}
ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_HIERARCHICAL_ACCESS_KEY }}
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
SNOWFLAKE_URL: ${{ vars.SNOWFLAKE_URL }}
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
SNOWFLAKE_CATALOG_SCHEMA: ${{ vars.SNOWFLAKE_CATALOG_SCHEMA }}
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
SNOWFLAKE_CATALOG_S3_ACCESS_KEY_ID: ${{ vars.SNOWFLAKE_CATALOG_S3_ACCESS_KEY_ID }}
SNOWFLAKE_CATALOG_S3_SECRET_ACCESS_KEY: ${{ secrets.SNOWFLAKE_CATALOG_S3_SECRET_ACCESS_KEY }}
SNOWFLAKE_EXTERNAL_VOLUME: ${{ vars.SNOWFLAKE_EXTERNAL_VOLUME }}
SNOWFLAKE_CATALOG_S3_REGION: ${{ vars.SNOWFLAKE_CATALOG_S3_REGION }}
if: >-
contains(matrix.modules, 'trino-iceberg') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '' || env.GCP_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-iceberg ${{ format('-P {0}', matrix.profile) }} \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.gcp-credentials-key="${GCP_CREDENTIALS_KEY}" \
-Dtesting.azure-abfs-container="${ABFS_CONTAINER}" \
-Dtesting.azure-abfs-account="${ABFS_ACCOUNT}" \
-Dtesting.azure-abfs-access-key="${ABFS_ACCESS_KEY}" \
-Dtesting.snowflake.catalog.user="${SNOWFLAKE_USER}" \
-Dtesting.snowflake.catalog.password="${SNOWFLAKE_PASSWORD}" \
-Dtesting.snowflake.catalog.account-url="${SNOWFLAKE_URL}" \
-Dtesting.snowflake.catalog.database="${SNOWFLAKE_DATABASE}" \
-Dtesting.snowflake.catalog.schema="${SNOWFLAKE_CATALOG_SCHEMA}" \
-Dtesting.snowflake.catalog.role="${SNOWFLAKE_ROLE}" \
-Dtesting.snowflake.catalog.warehouse="${SNOWFLAKE_WAREHOUSE}" \
-Dtesting.snowflake.catalog.s3.access-key="${SNOWFLAKE_CATALOG_S3_ACCESS_KEY_ID}" \
-Dtesting.snowflake.catalog.s3.secret-key="${SNOWFLAKE_CATALOG_S3_SECRET_ACCESS_KEY}" \
-Dtesting.snowflake.catalog.s3.external.volume="${SNOWFLAKE_EXTERNAL_VOLUME}" \
-Dtesting.snowflake.catalog.s3.region="${SNOWFLAKE_CATALOG_S3_REGION}"
- name: Cloud Redshift Tests ${{ matrix.profile }}
id: tests-redshift
env:
AWS_REGION: ${{ vars.REDSHIFT_AWS_REGION }}
AWS_ACCESS_KEY_ID: ${{ vars.REDSHIFT_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_AWS_SECRET_ACCESS_KEY }}
REDSHIFT_SUBNET_GROUP_NAME: ${{ vars.REDSHIFT_SUBNET_GROUP_NAME }}
REDSHIFT_IAM_ROLES: ${{ vars.REDSHIFT_IAM_ROLES }}
REDSHIFT_VPC_SECURITY_GROUP_IDS: ${{ vars.REDSHIFT_VPC_SECURITY_GROUP_IDS }}
REDSHIFT_S3_TPCH_TABLES_ROOT: ${{ vars.REDSHIFT_S3_TPCH_TABLES_ROOT }}
REDSHIFT_S3_UNLOAD_ROOT: ${{ vars.REDSHIFT_S3_UNLOAD_ROOT }}
if: >-
contains(matrix.modules, 'trino-redshift') &&
(contains(matrix.profile, 'cloud-tests') || contains(matrix.profile, 'fte-tests')) &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.AWS_ACCESS_KEY_ID != '' || env.REDSHIFT_SUBNET_GROUP_NAME != '')
run: |
source .github/bin/redshift/setup-aws-redshift.sh
$MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ format('-P {0}', matrix.profile) }} \
-Dtest.redshift.jdbc.user="${REDSHIFT_USER}" \
-Dtest.redshift.jdbc.password="${REDSHIFT_PASSWORD}" \
-Dtest.redshift.jdbc.endpoint="${REDSHIFT_ENDPOINT}:${REDSHIFT_PORT}/" \
-Dtest.redshift.s3.tpch.tables.root="${REDSHIFT_S3_TPCH_TABLES_ROOT}" \
-Dtest.redshift.s3.unload.root="${REDSHIFT_S3_UNLOAD_ROOT}" \
-Dtest.redshift.iam.role="${REDSHIFT_IAM_ROLES}" \
-Dtest.redshift.aws.region="${AWS_REGION}" \
-Dtest.redshift.aws.access-key="${AWS_ACCESS_KEY_ID}" \
-Dtest.redshift.aws.secret-key="${AWS_SECRET_ACCESS_KEY}"
- name: Cleanup ephemeral Redshift Cluster
env:
AWS_REGION: ${{ vars.REDSHIFT_AWS_REGION }}
AWS_ACCESS_KEY_ID: ${{ vars.REDSHIFT_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_AWS_SECRET_ACCESS_KEY }}
# Cancelled workflows may have left the ephemeral cluster running
if: always()
run: .github/bin/redshift/delete-aws-redshift.sh
- name: Teradata Tests
id: tests-teradata
env:
CLEARSCAPE_TOKEN: ${{ secrets.CLEARSCAPE_TOKEN }}
CLEARSCAPE_PASSWORD: ${{ secrets.CLEARSCAPE_PASSWORD }}
CLEARSCAPE_REGION: ${{ vars.CLEARSCAPE_REGION }}
if: matrix.modules == 'plugin/trino-teradata' && contains(matrix.profile, 'clearscape-tests') && (env.CLEARSCAPE_TOKEN != '' || env.CLEARSCAPE_PASSWORD != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-teradata -Pclearscape-tests
- name: Sanitize artifact name
if: always()
run: |
# Generate a valid artifact name and make it available to next steps as
# an environment variable ARTIFACT_NAME
# ", :, <, >, |, *, ?, \, / are not allowed in artifact names, replace it with an underscore
name=$(echo -n "${{ matrix.modules }}, ${{ matrix.profile }}, ${{ matrix.jdk }}" | sed -e 's/[":<>|\*\?\\\/]/_/g')
# final artifact name can't be longer than 128 characters
echo "ARTIFACT_NAME=${name:0:100}" >> $GITHUB_ENV
- name: Upload test results
uses: ./.github/actions/process-test-results
if: always()
with:
artifact-name: ${{ env.ARTIFACT_NAME }}
has-failed-tests: >-
${{ steps.tests.outcome == 'failure'
|| steps.tests-hdfs-isolated.outcome == 'failure'
|| steps.tests-hdfs.outcome == 'failure'
|| steps.tests-s3.outcome == 'failure'
|| steps.tests-azure.outcome == 'failure'
|| steps.tests-gcs.outcome == 'failure'
|| steps.tests-delta.outcome == 'failure'
|| steps.tests-bq.outcome == 'failure'
|| steps.tests-bq-ci.outcome == 'failure'
|| steps.tests-bq-smoke.outcome == 'failure'
|| steps.tests-iceberg.outcome == 'failure'
|| steps.tests-redshift.outcome == 'failure'
|| steps.tests-snowflake.outcome == 'failure'
}}
upload-heap-dump: ${{ env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
- name: Update PR check
uses: ./.github/actions/update-check
if: >-
failure() &&
github.event_name == 'repository_dispatch' &&
github.event.client_payload.slash_command.args.named.sha != '' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha
with:
pull_request_number: ${{ github.event.client_payload.pull_request.number }}
check_name: ${{ github.job }} with secrets
conclusion: ${{ job.status }}
github_token: ${{ secrets.GITHUB_TOKEN }}
build-success:
if: ${{ always() }} # if `failure()` would not work for cancellations, `!success()` would not work for skipped jobs
runs-on: ubuntu-latest
needs:
- build-test-matrix
- error-prone-checks
- maven-checks
- path-filters
- test
steps:
- name: "Check results"
run: |
# generated by TestCiWorkflow
echo '${{ needs.artifact-checks.result }}' | grep -xE 'success|skipped' || { echo 'Job "artifact-checks" failed' >&2; exit 1; }
echo '${{ needs.build-pt.result }}' | grep -xE 'success|skipped' || { echo 'Job "build-pt" failed' >&2; exit 1; }
echo '${{ needs.build-test-matrix.result }}' | grep -xE 'success|skipped' || { echo 'Job "build-test-matrix" failed' >&2; exit 1; }
echo '${{ needs.check-commit.result }}' | grep -xE 'success|skipped' || { echo 'Job "check-commit" failed' >&2; exit 1; }
echo '${{ needs.check-commits-dispatcher.result }}' | grep -xE 'success|skipped' || { echo 'Job "check-commits-dispatcher" failed' >&2; exit 1; }
echo '${{ needs.error-prone-checks.result }}' | grep -xE 'success|skipped' || { echo 'Job "error-prone-checks" failed' >&2; exit 1; }
echo '${{ needs.hive-tests.result }}' | grep -xE 'success|skipped' || { echo 'Job "hive-tests" failed' >&2; exit 1; }
echo '${{ needs.maven-checks.result }}' | grep -xE 'success|skipped' || { echo 'Job "maven-checks" failed' >&2; exit 1; }
echo '${{ needs.path-filters.result }}' | grep -xE 'success|skipped' || { echo 'Job "path-filters" failed' >&2; exit 1; }
echo '${{ needs.pt.result }}' | grep -xE 'success|skipped' || { echo 'Job "pt" failed' >&2; exit 1; }
echo '${{ needs.test.result }}' | grep -xE 'success|skipped' || { echo 'Job "test" failed' >&2; exit 1; }
echo '${{ needs.test-jdbc-compatibility.result }}' | grep -xE 'success|skipped' || { echo 'Job "test-jdbc-compatibility" failed' >&2; exit 1; }
echo '${{ needs.test-other-modules.result }}' | grep -xE 'success|skipped' || { echo 'Job "test-other-modules" failed' >&2; exit 1; }